db-sync-tool-kmi 2.11.6__py3-none-any.whl → 3.0.2__py3-none-any.whl

db_sync_tool/remote/rsync.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 rsync script
@@ -7,6 +6,7 @@ rsync script
 
 import re
 from db_sync_tool.utility import mode, system, output
+from db_sync_tool.utility.console import get_output_manager
 
 # Default options for rsync command
 # https://wiki.ubuntuusers.de/rsync/
@@ -31,8 +31,10 @@ def get_password_environment(client):
     if not client:
         return ''
 
-    if system.config['use_sshpass'] and not 'ssh_key' in system.config[client] and 'password' in system.config[client]:
-        return f'SSHPASS=\'{system.config[client]["password"]}\' '
+    cfg = system.get_typed_config()
+    client_cfg = cfg.get_client(client)
+    if cfg.use_sshpass and not client_cfg.ssh_key and client_cfg.password:
+        return f'SSHPASS=\'{client_cfg.password}\' '
     return ''
 
 
@@ -42,20 +44,19 @@ def get_authorization(client):
     :param client: String
     :return: String
     """
-    _ssh_key = None
     if not client:
         return ''
 
-    if 'ssh_key' in system.config[client]:
-        _ssh_key = system.config[mode.Client.ORIGIN]['ssh_key']
-
-    _ssh_port = system.config[client]['port'] if 'port' in system.config[client] else 22
+    cfg = system.get_typed_config()
+    client_cfg = cfg.get_client(client)
+    _ssh_key = client_cfg.ssh_key
+    _ssh_port = client_cfg.port
 
     if _ssh_key is None:
-        if system.config['use_sshpass'] and get_password_environment(client):
+        if cfg.use_sshpass and get_password_environment(client):
             # In combination with SSHPASS environment variable
             # https://www.redhat.com/sysadmin/ssh-automation-sshpass
-            return f'--rsh="sshpass -e ssh -p{_ssh_port} -o StrictHostKeyChecking=no -l {system.config[client]["user"]}"'
+            return f'--rsh="sshpass -e ssh -p{_ssh_port} -o StrictHostKeyChecking=no -l {client_cfg.user}"'
         else:
             return f'-e "ssh -p{_ssh_port} -o StrictHostKeyChecking=no"'
     else:
@@ -70,7 +71,9 @@ def get_host(client):
     :return: String
     """
     if mode.is_remote(client):
-        return f'{system.config[client]["user"]}@{system.config[client]["host"]}:'
+        cfg = system.get_typed_config()
+        client_cfg = cfg.get_client(client)
+        return f'{client_cfg.user}@{client_cfg.host}:'
     return ''
 
 
@@ -79,9 +82,10 @@ def get_options():
     Prepare rsync options with stored default options and provided addtional options
     :return: String
     """
+    cfg = system.get_typed_config()
     _options = f'{" ".join(default_options)}'
-    if not system.config['use_rsync_options'] is None:
-        _options += f'{system.config["use_rsync_options"]}'
+    if cfg.use_rsync_options is not None:
+        _options += f'{cfg.use_rsync_options}'
     return _options
 
 
@@ -91,8 +95,7 @@ def read_stats(stats):
     :param stats: String
     :return:
     """
-    if system.config['verbose']:
-        print(f'{output.Subject.DEBUG}{output.CliFormat.BLACK}{stats}{output.CliFormat.ENDC}')
+    get_output_manager().debug(stats)
 
     _file_size = parse_string(stats, r'Total transferred file size:\s*([\d.]+[MKG]?)')
 

db_sync_tool/remote/system.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 System script
 """
 
-import sys
 from db_sync_tool.utility import mode, output, helper
+from db_sync_tool.utility.exceptions import DbSyncError
 from db_sync_tool.remote import client as remote_client
 
 
@@ -38,7 +37,7 @@ def run_ssh_command(command, ssh_client=remote_client.ssh_client_origin, client=
 
     if err and exit_status != 0:
         helper.run_script(client=client, script='error')
-        sys.exit(output.message(output.Subject.ERROR, err, False))
+        raise DbSyncError(err)
     elif err:
         output.message(output.Subject.WARNING, err, True)
 

db_sync_tool/remote/transfer.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 Transfer script
 """
 
-import sys
 from db_sync_tool.utility import mode, system, helper, output
 from db_sync_tool.database import utility as database_utility
 from db_sync_tool.remote import utility, client, rsync
@@ -16,6 +14,7 @@ def transfer_origin_database_dump():
     Transfer the origin database dump files
     :return:
     """
+    cfg = system.get_typed_config()
     if not mode.is_import():
         if mode.get_sync_mode() == mode.SyncMode.RECEIVER:
             get_origin_database_dump(helper.get_dump_dir(mode.Client.TARGET))
@@ -31,7 +30,7 @@ def transfer_origin_database_dump():
             put_origin_database_dump(system.default_local_sync_path)
         elif mode.get_sync_mode() == mode.SyncMode.SYNC_REMOTE or mode.get_sync_mode() == mode.SyncMode.SYNC_LOCAL:
             system.check_target_configuration()
-        elif system.config['is_same_client']:
+        elif cfg.is_same_client:
             utility.remove_origin_database_dump(True)
     else:
         system.check_target_configuration()
@@ -43,6 +42,7 @@ def get_origin_database_dump(target_path):
     :param target_path: String
     :return:
     """
+    cfg = system.get_typed_config()
     output.message(
         output.Subject.ORIGIN,
         'Downloading database dump',
@@ -51,16 +51,16 @@ def get_origin_database_dump(target_path):
     if mode.get_sync_mode() != mode.SyncMode.PROXY:
         helper.check_and_create_dump_dir(mode.Client.TARGET, target_path)
 
-    if not system.config['dry_run']:
-        _remotepath = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz'
+    if not cfg.dry_run:
+        _remotepath = database_utility.get_dump_gz_path(mode.Client.ORIGIN)
         _localpath = target_path
 
-        if system.config['use_rsync']:
+        if cfg.use_rsync:
             rsync.run_rsync_command(
                 remote_client=mode.Client.ORIGIN,
                 origin_path=_remotepath,
                 target_path=_localpath,
-                origin_ssh=system.config[mode.Client.ORIGIN]['user'] + '@' + system.config[mode.Client.ORIGIN]['host']
+                origin_ssh=cfg.origin.user + '@' + cfg.origin.host
             )
         else:
             #
@@ -68,29 +68,46 @@ def get_origin_database_dump(target_path):
             # https://github.com/paramiko/paramiko/issues/60
             #
             sftp = get_sftp_client(client.ssh_client_origin)
-            sftp.get(helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz',
-                     target_path + database_utility.database_dump_file_name + '.tar.gz', download_status)
+            sftp.get(database_utility.get_dump_gz_path(mode.Client.ORIGIN),
+                     target_path + database_utility.database_dump_file_name + '.gz', download_status)
             sftp.close()
-            if not system.config['mute']:
-                print('')
 
     utility.remove_origin_database_dump()
 
 
+def _transfer_status(sent, size, direction, subject_override=None):
+    """
+    Print transfer progress status.
+
+    :param sent: Bytes transferred
+    :param size: Total bytes
+    :param direction: 'downloaded' or 'uploaded'
+    :param subject_override: Optional subject prefix override
+    """
+    cfg = system.get_typed_config()
+    if cfg.mute:
+        return
+
+    from db_sync_tool.utility.console import get_output_manager
+    output_manager = get_output_manager()
+
+    # Track transfer size for final summary
+    if sent == size:
+        output_manager.track_stat("size", size)
+
+    # Use OutputManager for progress display
+    msg = "Downloading" if direction == "downloaded" else "Uploading"
+    output_manager.progress(sent, size, msg)
+
+
 def download_status(sent, size):
     """
-    Printing the download status information
-    :param sent: Float
-    :param size: Float
-    :return:
+    Callback for SFTP download progress.
+
+    :param sent: Bytes transferred
+    :param size: Total bytes
     """
-    if not system.config['mute']:
-        sent_mb = round(float(sent) / 1024 / 1024, 1)
-        size = round(float(size) / 1024 / 1024, 1)
-        sys.stdout.write(
-            output.Subject.ORIGIN + output.CliFormat.BLACK + '[REMOTE]' + output.CliFormat.ENDC + " Status: {0} MB of {1} MB downloaded".
-            format(sent_mb, size, ))
-        sys.stdout.write('\r')
+    _transfer_status(sent, size, 'downloaded')
 
 
 def put_origin_database_dump(origin_path):
@@ -99,6 +116,7 @@ def put_origin_database_dump(origin_path):
     :param origin_path: String
     :return:
     """
+    cfg = system.get_typed_config()
     if mode.get_sync_mode() == mode.SyncMode.PROXY:
         _subject = output.Subject.LOCAL
     else:
@@ -111,16 +129,16 @@ def put_origin_database_dump(origin_path):
     )
     helper.check_and_create_dump_dir(mode.Client.TARGET, helper.get_dump_dir(mode.Client.TARGET))
 
-    if not system.config['dry_run']:
-        _localpath = origin_path + database_utility.database_dump_file_name + '.tar.gz'
+    if not cfg.dry_run:
+        _localpath = origin_path + database_utility.database_dump_file_name + '.gz'
         _remotepath = helper.get_dump_dir(mode.Client.TARGET) + '/'
 
-        if system.config['use_rsync']:
+        if cfg.use_rsync:
             rsync.run_rsync_command(
                 remote_client=mode.Client.TARGET,
                 origin_path=_localpath,
                 target_path=_remotepath,
-                target_ssh=system.config[mode.Client.TARGET]['user'] + '@' + system.config[mode.Client.TARGET]['host']
+                target_ssh=cfg.target.user + '@' + cfg.target.host
             )
         else:
             #
@@ -128,35 +146,21 @@ def put_origin_database_dump(origin_path):
             # https://github.com/paramiko/paramiko/issues/60
             #
             sftp = get_sftp_client(client.ssh_client_target)
-            sftp.put(origin_path + database_utility.database_dump_file_name + '.tar.gz',
-                     helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name + '.tar.gz',
+            sftp.put(origin_path + database_utility.database_dump_file_name + '.gz',
+                     database_utility.get_dump_gz_path(mode.Client.TARGET),
                      upload_status)
             sftp.close()
-            if not system.config['mute']:
-                print('')
 
 
 
 def upload_status(sent, size):
     """
-    Printing the upload status information
-    :param sent: Float
-    :param size: Float
-    :return:
-    """
-    if not system.config['mute']:
-        sent_mb = round(float(sent) / 1024 / 1024, 1)
-        size = round(float(size) / 1024 / 1024, 1)
+    Callback for SFTP upload progress.
 
-        if (mode.get_sync_mode() == mode.SyncMode.PROXY):
-            _subject = output.Subject.LOCAL
-        else:
-            _subject = output.Subject.ORIGIN + output.CliFormat.BLACK + '[LOCAL]' + output.CliFormat.ENDC
-
-        sys.stdout.write(
-            _subject + " Status: {0} MB of {1} MB uploaded".
-            format(sent_mb, size, ))
-        sys.stdout.write('\r')
+    :param sent: Bytes transferred
+    :param size: Total bytes
+    """
+    _transfer_status(sent, size, 'uploaded')
 
 
 def get_sftp_client(ssh_client):

db_sync_tool/remote/utility.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 
 """
 Utility script
@@ -9,7 +8,6 @@ import os
 import paramiko
 from db_sync_tool.utility import mode, system, helper, output
 from db_sync_tool.database import utility as database_utility
-from db_sync_tool.remote import client as remote_client
 
 
 def remove_origin_database_dump(keep_compressed_file=False):
@@ -24,34 +22,33 @@ def remove_origin_database_dump(keep_compressed_file=False):
         True
     )
 
-    if system.config['dry_run']:
+    cfg = system.get_typed_config()
+    if cfg.dry_run:
         return
 
-    _file_path = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name
-    if mode.is_origin_remote():
-        mode.run_command(
-            helper.get_command(mode.Client.ORIGIN, 'rm') + ' ' + _file_path,
-            mode.Client.ORIGIN
-        )
-        if not keep_compressed_file:
+    _gz_path = database_utility.get_dump_gz_path(mode.Client.ORIGIN)
+
+    # With streaming compression, only .gz file exists on origin (no separate .sql)
+    if not keep_compressed_file:
+        if mode.is_origin_remote():
             mode.run_command(
-                helper.get_command(mode.Client.ORIGIN, 'rm') + ' ' + _file_path + '.tar.gz',
+                helper.get_command(mode.Client.ORIGIN, 'rm') + ' ' + _gz_path,
                 mode.Client.ORIGIN
             )
-    else:
-        os.remove(_file_path)
-        if not keep_compressed_file:
-            os.remove(f'{_file_path}.tar.gz')
+        else:
+            if os.path.isfile(_gz_path):
+                os.remove(_gz_path)
 
     if keep_compressed_file:
-        if 'keep_dumps' in system.config[mode.Client.ORIGIN]:
+        origin_cfg = cfg.origin
+        if origin_cfg.keep_dumps is not None:
             helper.clean_up_dump_dir(mode.Client.ORIGIN,
                                      helper.get_dump_dir(mode.Client.ORIGIN) + '*',
-                                     system.config[mode.Client.ORIGIN]['keep_dumps'])
+                                     origin_cfg.keep_dumps)
 
         output.message(
             output.Subject.INFO,
-            f'Database dump file is saved to: {_file_path}.tar.gz',
+            f'Database dump file is saved to: {_gz_path}',
             True,
             True
         )
@@ -62,17 +59,22 @@ def remove_target_database_dump():
     Removing the target database dump files
     :return:
     """
-    _file_path = helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name
+    cfg = system.get_typed_config()
+    _file_path = database_utility.get_dump_file_path(mode.Client.TARGET)
+    _gz_file_path = database_utility.get_dump_gz_path(mode.Client.TARGET)
 
     #
     # Move dump to specified directory
     #
-    if system.config['keep_dump']:
+    if cfg.keep_dump:
         helper.create_local_temporary_data_dir()
-        _keep_dump_path = system.default_local_sync_path + database_utility.database_dump_file_name
+        # Copy the .gz file (streaming compression means only .gz exists)
+        # database_dump_file_name is guaranteed to be set at this point
+        _dump_name = database_utility.database_dump_file_name or ''
+        _keep_dump_path = system.default_local_sync_path + _dump_name + '.gz'
         mode.run_command(
             helper.get_command('target',
-                               'cp') + ' ' + _file_path + ' ' + _keep_dump_path,
+                               'cp') + ' ' + _gz_file_path + ' ' + _keep_dump_path,
             mode.Client.TARGET
         )
         output.message(
@@ -92,23 +94,20 @@ def remove_target_database_dump():
             True
         )
 
-        if system.config['dry_run']:
+        if cfg.dry_run:
             return
 
         if mode.is_target_remote():
+            # Remove both decompressed .sql and compressed .gz
             mode.run_command(
-                helper.get_command(mode.Client.TARGET, 'rm') + ' ' + _file_path,
-                mode.Client.TARGET
-            )
-            mode.run_command(
-                helper.get_command(mode.Client.TARGET, 'rm') + ' ' + _file_path + '.tar.gz',
+                helper.get_command(mode.Client.TARGET, 'rm') + ' -f ' + _file_path + ' ' + _gz_file_path,
                 mode.Client.TARGET
             )
         else:
             if os.path.isfile(_file_path):
                 os.remove(_file_path)
-            if os.path.isfile(f'{_file_path}.tar.gz'):
-                os.remove(f'{_file_path}.tar.gz')
+            if os.path.isfile(_gz_file_path):
+                os.remove(_gz_file_path)
 
 
 def check_keys_from_ssh_agent():

db_sync_tool/sync.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
-# -*- coding: future_fstrings -*-
 """
 Sync script
 """
 
-from db_sync_tool.utility import system, helper, info
-from db_sync_tool.database import process
-from db_sync_tool.remote import transfer, client as remote_client
+import sys
+from db_sync_tool.utility import system, helper, info, output
+from db_sync_tool.utility.exceptions import DbSyncError
+from db_sync_tool.database import process, utility as database_utility
+from db_sync_tool.remote import transfer, client as remote_client, file_transfer
 
 
 class Sync:
@@ -29,6 +30,8 @@ class Sync:
                  use_rsync=False,
                  use_rsync_options=None,
                  reverse=False,
+                 with_files=False,
+                 files_only=False,
                  config=None,
                  args=None):
         """
@@ -47,34 +50,55 @@ class Sync:
         :param use_rsync:
         :param use_rsync_options:
         :param reverse:
+        :param with_files:
+        :param files_only:
         :param config:
         :param args:
         """
         if config is None:
             config = {}
 
-        info.print_header(mute)
-        system.check_args_options(
-            config_file,
-            verbose,
-            yes,
-            mute,
-            dry_run,
-            import_file,
-            dump_name,
-            keep_dump,
-            host_file,
-            clear,
-            force_password,
-            use_rsync,
-            use_rsync_options,
-            reverse
-        )
-        system.get_configuration(config, args)
-        system.check_authorizations()
-        process.create_origin_database_dump()
-        transfer.transfer_origin_database_dump()
-        process.import_database_dump()
-        helper.clean_up()
-        remote_client.close_ssh_clients()
+        info.print_header(mute, verbose)
+        try:
+            system.check_args_options(
+                config_file,
+                verbose,
+                yes,
+                mute,
+                dry_run,
+                import_file,
+                dump_name,
+                keep_dump,
+                host_file,
+                clear,
+                force_password,
+                use_rsync,
+                use_rsync_options,
+                reverse,
+                with_files,
+                files_only
+            )
+            system.get_configuration(config, args)
+            system.check_authorizations()
+
+            cfg = system.get_typed_config()
+
+            # Database sync (skip if --files-only)
+            if not cfg.files_only:
+                process.create_origin_database_dump()
+                transfer.transfer_origin_database_dump()
+                process.import_database_dump()
+
+            # File sync (only if --with-files or --files-only)
+            file_transfer.transfer_files()
+
+            helper.clean_up()
+        except DbSyncError as e:
+            output.message(output.Subject.ERROR, str(e), do_print=True)
+            sys.exit(1)
+        finally:
+            # Always clean up sensitive credential files, even on error
+            database_utility.cleanup_mysql_config_files()
+            remote_client.close_ssh_clients()
+            file_transfer.cleanup()
         info.print_footer()