testgres-pg-probackup2 0.0.1__py3-none-any.whl

pg_probackup2/gdb.py

@@ -0,0 +1,346 @@
+import functools
+import os
+import re
+import subprocess
+import sys
+import unittest
+from time import sleep
+
+
+class GdbException(Exception):
+    def __init__(self, message="False"):
+        self.message = message
+
+    def __str__(self):
+        return '\n ERROR: {0}\n'.format(repr(self.message))
+
+
+class GDBobj:
+    _gdb_enabled = False
+    _gdb_ok = False
+    _gdb_ptrace_ok = False
+
+    def __init__(self, cmd, env, attach=False):
+        self.verbose = env.verbose
+        self.output = ''
+        self._did_quit = False
+        self.has_breakpoint = False
+
+        # Check gdb flag is set up
+        if not hasattr(env, "_gdb_decorated") or not env._gdb_decorated:
+            raise GdbException("Test should be decorated with @needs_gdb")
+        if not self._gdb_enabled:
+            raise GdbException("No `PGPROBACKUP_GDB=on` is set.")
+        if not self._gdb_ok:
+            if not self._gdb_ptrace_ok:
+                raise GdbException("set /proc/sys/kernel/yama/ptrace_scope to 0"
+                                   " to run GDB tests")
+            raise GdbException("No gdb usage possible.")
+
+        # Check gdb presense
+        try:
+            gdb_version, _ = subprocess.Popen(
+                ['gdb', '--version'],
+                stdout=subprocess.PIPE
+            ).communicate()
+        except OSError:
+            raise GdbException("Couldn't find gdb on the path")
+
+        self.base_cmd = [
+            'gdb',
+            '--interpreter',
+            'mi2',
+        ]
+
+        if attach:
+            self.cmd = self.base_cmd + ['--pid'] + cmd
+        else:
+            self.cmd = self.base_cmd + ['--args'] + cmd
+
+        # Get version
+        gdb_version_number = re.search(
+            br"^GNU gdb [^\d]*(\d+)\.(\d)",
+            gdb_version)
+        self.major_version = int(gdb_version_number.group(1))
+        self.minor_version = int(gdb_version_number.group(2))
+
+        if self.verbose:
+            print([' '.join(map(str, self.cmd))])
+
+        self.proc = subprocess.Popen(
+            self.cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            bufsize=0,
+            text=True,
+            errors='replace',
+        )
+        self.gdb_pid = self.proc.pid
+
+        while True:
+            line = self.get_line()
+
+            if 'No such process' in line:
+                raise GdbException(line)
+
+            if not line.startswith('(gdb)'):
+                pass
+            else:
+                break
+
+    def __del__(self):
+        if not self._did_quit and hasattr(self, "proc"):
+            try:
+                self.quit()
+            except subprocess.TimeoutExpired:
+                self.kill()
+
+    def get_line(self):
+        line = self.proc.stdout.readline()
+        self.output += line
+        return line
+
+    def kill(self):
+        self._did_quit = True
+        self.proc.kill()
+        self.proc.wait(3)
+        self.proc.stdin.close()
+        self.proc.stdout.close()
+
+    def set_breakpoint(self, location):
+
+        result = self._execute('break ' + location)
+        self.has_breakpoint = True
+        for line in result:
+            if line.startswith('~"Breakpoint'):
+                return
+
+            elif line.startswith('=breakpoint-created'):
+                return
+
+            elif line.startswith('^error'):  # or line.startswith('(gdb)'):
+                break
+
+            elif line.startswith('&"break'):
+                pass
+
+            elif line.startswith('&"Function'):
+                raise GdbException(line)
+
+            elif line.startswith('&"No line'):
+                raise GdbException(line)
+
+            elif line.startswith('~"Make breakpoint pending on future shared'):
+                raise GdbException(line)
+
+        raise GdbException(
+            'Failed to set breakpoint.\n Output:\n {0}'.format(result)
+        )
+
+    def remove_all_breakpoints(self):
+        if not self.has_breakpoint:
+            return
+
+        result = self._execute('delete')
+        self.has_breakpoint = False
+        for line in result:
+
+            if line.startswith('^done'):
+                return
+
+        raise GdbException(
+            'Failed to remove breakpoints.\n Output:\n {0}'.format(result)
+        )
+
+    def run_until_break(self):
+        result = self._execute('run', False)
+        for line in result:
+            if line.startswith('*stopped,reason="breakpoint-hit"'):
+                return
+        raise GdbException(
+            'Failed to run until breakpoint.\n'
+        )
+
+    def continue_execution_until_running(self):
+        result = self._execute('continue')
+
+        for line in result:
+            if line.startswith('*running') or line.startswith('^running'):
+                return
+            if line.startswith('*stopped,reason="breakpoint-hit"'):
+                continue
+            if line.startswith('*stopped,reason="exited-normally"'):
+                continue
+
+        raise GdbException(
+            'Failed to continue execution until running.\n'
+        )
+
+    def signal(self, sig):
+        if 'KILL' in sig:
+            self.remove_all_breakpoints()
+        self._execute(f'signal {sig}')
+
+    def continue_execution_until_exit(self):
+        self.remove_all_breakpoints()
+        result = self._execute('continue', False)
+
+        for line in result:
+            if line.startswith('*running'):
+                continue
+            if line.startswith('*stopped,reason="breakpoint-hit"'):
+                continue
+            if line.startswith('*stopped,reason="exited') or line == '*stopped\n':
+                self.quit()
+                return
+
+        raise GdbException(
+            'Failed to continue execution until exit.\n'
+        )
+
+    def continue_execution_until_error(self):
+        self.remove_all_breakpoints()
+        result = self._execute('continue', False)
+
+        for line in result:
+            if line.startswith('^error'):
+                return
+            if line.startswith('*stopped,reason="exited'):
+                return
+            if line.startswith(
+                    '*stopped,reason="signal-received",signal-name="SIGABRT"'):
+                return
+
+        raise GdbException(
+            'Failed to continue execution until error.\n')
+
+    def continue_execution_until_break(self, ignore_count=0):
+        if ignore_count > 0:
+            result = self._execute(
+                'continue ' + str(ignore_count),
+                False
+            )
+        else:
+            result = self._execute('continue', False)
+
+        for line in result:
+            if line.startswith('*stopped,reason="breakpoint-hit"'):
+                return
+            if line.startswith('*stopped,reason="exited-normally"'):
+                break
+
+        raise GdbException(
+            'Failed to continue execution until break.\n')
+
+    def show_backtrace(self):
+        return self._execute("backtrace", running=False)
+
+    def stopped_in_breakpoint(self):
+        while True:
+            line = self.get_line()
+            if self.verbose:
+                print(line)
+            if line.startswith('*stopped,reason="breakpoint-hit"'):
+                return True
+
+    def detach(self):
+        if not self._did_quit:
+            self._execute('detach')
+
+    def quit(self):
+        if not self._did_quit:
+            self._did_quit = True
+            self.proc.terminate()
+            self.proc.wait(3)
+            self.proc.stdin.close()
+            self.proc.stdout.close()
+
+    # use for breakpoint, run, continue
+    def _execute(self, cmd, running=True):
+        output = []
+        self.proc.stdin.flush()
+        self.proc.stdin.write(cmd + '\n')
+        self.proc.stdin.flush()
+        sleep(1)
+
+        # look for command we just send
+        while True:
+            line = self.get_line()
+            if self.verbose:
+                print(repr(line))
+
+            if cmd not in line:
+                continue
+            else:
+                break
+
+        while True:
+            line = self.get_line()
+            output += [line]
+            if self.verbose:
+                print(repr(line))
+            if line.startswith('^done') or line.startswith('*stopped'):
+                break
+            if line.startswith('^error'):
+                break
+            if running and (line.startswith('*running') or line.startswith('^running')):
+                #            if running and line.startswith('*running'):
+                break
+        return output
+
+
+def _set_gdb(self):
+    test_env = os.environ.copy()
+    self._gdb_enabled = test_env.get('PGPROBACKUP_GDB') == 'ON'
+    self._gdb_ok = self._gdb_enabled
+    if not self._gdb_enabled or sys.platform != 'linux':
+        return
+    try:
+        with open('/proc/sys/kernel/yama/ptrace_scope') as f:
+            ptrace = f.read()
+    except FileNotFoundError:
+        self._gdb_ptrace_ok = True
+        return
+    self._gdb_ptrace_ok = int(ptrace) == 0
+    self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok
+
+
+def _check_gdb_flag_or_skip_test():
+    if not GDBobj._gdb_enabled:
+        return ("skip",
+                "Specify PGPROBACKUP_GDB and build without "
+                "optimizations for run this test"
+                )
+    if GDBobj._gdb_ok:
+        return None
+    if not GDBobj._gdb_ptrace_ok:
+        return ("fail", "set /proc/sys/kernel/yama/ptrace_scope to 0"
+                        " to run GDB tests")
+    else:
+        return ("fail", "use of gdb is not possible")
+
+
+def needs_gdb(func):
+    check = _check_gdb_flag_or_skip_test()
+    if not check:
+        @functools.wraps(func)
+        def ok_wrapped(self):
+            self._gdb_decorated = True
+            func(self)
+
+        return ok_wrapped
+    reason = check[1]
+    if check[0] == "skip":
+        return unittest.skip(reason)(func)
+    elif check[0] == "fail":
+        @functools.wraps(func)
+        def fail_wrapper(self):
+            self.fail(reason)
+
+        return fail_wrapper
+    else:
+        raise "Wrong action {0}".format(check)
+
+
+_set_gdb(GDBobj)

pg_probackup2/init_helpers.py

@@ -0,0 +1,207 @@
+from functools import reduce
+import getpass
+import os
+import re
+import shutil
+import subprocess
+import sys
+import testgres
+
+try:
+    import lz4.frame  # noqa: F401
+
+    HAVE_LZ4 = True
+except ImportError as e:
+    HAVE_LZ4 = False
+    LZ4_error = e
+
+try:
+    import zstd  # noqa: F401
+
+    HAVE_ZSTD = True
+except ImportError as e:
+    HAVE_ZSTD = False
+    ZSTD_error = e
+
+delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y']
+
+try:
+    testgres.configure_testgres(
+        cache_initdb=False,
+        cached_initdb_dir=False,
+        node_cleanup_full=delete_logs)
+except Exception as e:
+    print("Can't configure testgres: {0}".format(e))
+
+
+class Init(object):
+    def __init__(self):
+        if '-v' in sys.argv or '--verbose' in sys.argv:
+            self.verbose = True
+        else:
+            self.verbose = False
+
+        self._pg_config = testgres.get_pg_config()
+        self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise'
+        self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman'
+        self.is_pgpro = 'PGPRO_EDITION' in self._pg_config
+        self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE']
+        self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS']
+        version = self._pg_config['VERSION'].rstrip('develalphabetapre')
+        parts = [*version.split(' ')[1].split('.'), '0', '0'][:3]
+        parts[0] = re.match(r'\d+', parts[0]).group()
+        self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0)
+
+        test_env = os.environ.copy()
+        envs_list = [
+            'LANGUAGE',
+            'LC_ALL',
+            'PGCONNECT_TIMEOUT',
+            'PGDATA',
+            'PGDATABASE',
+            'PGHOSTADDR',
+            'PGREQUIRESSL',
+            'PGSERVICE',
+            'PGSSLMODE',
+            'PGUSER',
+            'PGPORT',
+            'PGHOST'
+        ]
+
+        for e in envs_list:
+            test_env.pop(e, None)
+
+        test_env['LC_MESSAGES'] = 'C'
+        test_env['LC_TIME'] = 'C'
+        self._test_env = test_env
+
+        # Get the directory from which the script was executed
+        self.source_path = os.getcwd()
+        tmp_path = test_env.get('PGPROBACKUP_TMP_DIR')
+        if tmp_path and os.path.isabs(tmp_path):
+            self.tmp_path = tmp_path
+        else:
+            self.tmp_path = os.path.abspath(
+                os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs'))
+            )
+
+        os.makedirs(self.tmp_path, exist_ok=True)
+
+        self.username = getpass.getuser()
+
+        self.probackup_path = None
+        if 'PGPROBACKUPBIN' in test_env:
+            if shutil.which(test_env["PGPROBACKUPBIN"]):
+                self.probackup_path = test_env["PGPROBACKUPBIN"]
+            else:
+                if self.verbose:
+                    print('PGPROBACKUPBIN is not an executable file')
+
+        if not self.probackup_path:
+            probackup_path_tmp = os.path.join(
+                testgres.get_pg_config()['BINDIR'], 'pg_probackup')
+
+            if os.path.isfile(probackup_path_tmp):
+                if not os.access(probackup_path_tmp, os.X_OK):
+                    print('{0} is not an executable file'.format(
+                        probackup_path_tmp))
+                else:
+                    self.probackup_path = probackup_path_tmp
+
+        if not self.probackup_path:
+            probackup_path_tmp = self.source_path
+
+            if os.path.isfile(probackup_path_tmp):
+                if not os.access(probackup_path_tmp, os.X_OK):
+                    print('{0} is not an executable file'.format(
+                        probackup_path_tmp))
+                else:
+                    self.probackup_path = probackup_path_tmp
+
+        if not self.probackup_path:
+            print('pg_probackup binary is not found')
+            exit(1)
+
+        if os.name == 'posix':
+            self.EXTERNAL_DIRECTORY_DELIMITER = ':'
+            os.environ['PATH'] = os.path.dirname(
+                self.probackup_path) + ':' + os.environ['PATH']
+
+        elif os.name == 'nt':
+            self.EXTERNAL_DIRECTORY_DELIMITER = ';'
+            os.environ['PATH'] = os.path.dirname(
+                self.probackup_path) + ';' + os.environ['PATH']
+
+        self.probackup_old_path = None
+        if 'PGPROBACKUPBIN_OLD' in test_env:
+            if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)):
+                self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD']
+            else:
+                if self.verbose:
+                    print('PGPROBACKUPBIN_OLD is not an executable file')
+
+        self.probackup_version = None
+        self.old_probackup_version = None
+
+        probackup_version_output = subprocess.check_output(
+            [self.probackup_path, "--version"],
+            stderr=subprocess.STDOUT,
+        ).decode('utf-8')
+        self.probackup_version = re.search(r"\d+\.\d+\.\d+",
+                                           probackup_version_output
+                                           ).group(0)
+        compressions = re.search(r"\(compressions: ([^)]*)\)",
+                                 probackup_version_output).group(1)
+        self.probackup_compressions = {s.strip() for s in compressions.split(',')}
+
+        if self.probackup_old_path:
+            old_probackup_version_output = subprocess.check_output(
+                [self.probackup_old_path, "--version"],
+                stderr=subprocess.STDOUT,
+            ).decode('utf-8')
+            self.old_probackup_version = re.search(r"\d+\.\d+\.\d+",
+                                                   old_probackup_version_output
+                                                   ).group(0)
+
+        self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON'
+        self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000
+
+        self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON'
+        env_compress = test_env.get('ARCHIVE_COMPRESSION', None)
+        if env_compress:
+            env_compress = env_compress.lower()
+        if env_compress in ('on', 'zlib'):
+            self.compress_suffix = '.gz'
+            self.archive_compress = 'zlib'
+        elif env_compress == 'lz4':
+            if not HAVE_LZ4:
+                raise LZ4_error
+            if 'lz4' not in self.probackup_compressions:
+                raise Exception("pg_probackup is not compiled with lz4 support")
+            self.compress_suffix = '.lz4'
+            self.archive_compress = 'lz4'
+        elif env_compress == 'zstd':
+            if not HAVE_ZSTD:
+                raise ZSTD_error
+            if 'zstd' not in self.probackup_compressions:
+                raise Exception("pg_probackup is not compiled with zstd support")
+            self.compress_suffix = '.zst'
+            self.archive_compress = 'zstd'
+        else:
+            self.compress_suffix = ''
+            self.archive_compress = False
+
+        cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None)
+        if cfs_compress:
+            self.cfs_compress = cfs_compress.lower()
+        else:
+            self.cfs_compress = self.archive_compress
+
+        os.environ["PGAPPNAME"] = "pg_probackup"
+        self.delete_logs = delete_logs
+
+    def test_env(self):
+        return self._test_env.copy()
+
+
+init_params = Init()

pg_probackup2/storage/fs_backup.py

@@ -0,0 +1,101 @@
+"""
+Utilities for accessing pg_probackup backup data on file system.
+"""
+import os
+import shutil
+
+from ..init_helpers import init_params
+
+
+class TestBackupDir:
+
+    def list_instance_backups(self, instance):
+        raise NotImplementedError()
+
+    def list_files(self, sub_dir, recursive=False):
+        raise NotImplementedError()
+
+    def list_dirs(self, sub_dir):
+        raise NotImplementedError()
+
+    def read_file(self, sub_path, *, text=True):
+        raise NotImplementedError()
+
+    def write_file(self, sub_path, data, *, text=True):
+        raise NotImplementedError()
+
+    def cleanup(self):
+        raise NotImplementedError()
+
+    def remove_file(self, sub_path):
+        raise NotImplementedError()
+
+    def remove_dir(self, sub_path):
+        raise NotImplementedError()
+
+    def exists(self, sub_path):
+        raise NotImplementedError()
+
+
+class FSTestBackupDir(TestBackupDir):
+    is_file_based = True
+
+    """ Backup directory. Usually created by running pg_probackup init -B <path>"""
+
+    def __init__(self, *, rel_path, backup):
+        self.path = os.path.join(init_params.tmp_path, rel_path, backup)
+        self.pb_args = ('-B', self.path)
+
+    def list_instance_backups(self, instance):
+        full_path = os.path.join(self.path, 'backups', instance)
+        return sorted((x for x in os.listdir(full_path)
+                      if os.path.isfile(os.path.join(full_path, x, 'backup.control'))))
+
+    def list_files(self, sub_dir, recursive=False):
+        full_path = os.path.join(self.path, sub_dir)
+        if not recursive:
+            return [f for f in os.listdir(full_path)
+                    if os.path.isfile(os.path.join(full_path, f))]
+        files = []
+        for rootdir, dirs, files_in_dir in os.walk(full_path):
+            rootdir = rootdir[len(self.path) + 1:]
+            files.extend(os.path.join(rootdir, file) for file in files_in_dir)
+        return files
+
+    def list_dirs(self, sub_dir):
+        full_path = os.path.join(self.path, sub_dir)
+        return [f for f in os.listdir(full_path)
+                if os.path.isdir(os.path.join(full_path, f))]
+
+    def read_file(self, sub_path, *, text=True):
+        full_path = os.path.join(self.path, sub_path)
+        with open(full_path, 'r' if text else 'rb') as fin:
+            return fin.read()
+
+    def write_file(self, sub_path, data, *, text=True):
+        full_path = os.path.join(self.path, sub_path)
+        with open(full_path, 'w' if text else 'wb') as fout:
+            fout.write(data)
+
+    def cleanup(self):
+        shutil.rmtree(self.path, ignore_errors=True)
+
+    def remove_file(self, sub_path):
+        os.remove(os.path.join(self.path, sub_path))
+
+    def remove_dir(self, sub_path):
+        full_path = os.path.join(self.path, sub_path)
+        shutil.rmtree(full_path, ignore_errors=True)
+
+    def exists(self, sub_path):
+        full_path = os.path.join(self.path, sub_path)
+        return os.path.exists(full_path)
+
+    def __str__(self):
+        return self.path
+
+    def __repr__(self):
+        return "FSTestBackupDir" + str(self.path)
+
+    def __fspath__(self):
+        return self.path