testgres-pg-probackup2 0.0.1__tar.gz → 0.0.3__tar.gz

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: testgres_pg_probackup2
-Version: 0.0.1
+Version: 0.0.3
 Summary: Plugin for testgres that manages pg_probackup2
 Home-page: https://github.com/postgrespro/testgres
 Author: Postgres Professional

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/README.md

@@ -16,22 +16,14 @@ The package requires testgres~=1.9.3.
 
 ## Usage
 
-### Environment
+### Environment variables
 
-> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`.
-
-There are several ways to specify a custom postgres installation:
-
-* export `PG_CONFIG` environment variable pointing to the `pg_config` executable;
-* export `PG_BIN` environment variable pointing to the directory with executable files.
-
-Example:
-
-```bash
-export PG_BIN=$HOME/pg/bin
-python my_tests.py
-```
+| Variable | Required | Default value | Description |
+| - | - | - | - |
+| PGPROBACKUP_TMP_DIR | No | tests/tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the current working directory. |
+| PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. |
 
+See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables.
 
 ### Examples
 

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/pg_probackup2/app.py

@@ -4,7 +4,6 @@ import json
 import os
 import re
 import subprocess
-import sys
 import threading
 import time
 import unittest
@@ -37,6 +36,10 @@ class ProbackupException(Exception):
         return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd)
 
 
+# Local backup control
+fs_backup_class = FSTestBackupDir
+
+
 class ProbackupApp:
 
     def __init__(self, test_class: unittest.TestCase,
@@ -53,6 +56,7 @@ class ProbackupApp:
         self.verbose = init_params.verbose
         self.archive_compress = init_params.archive_compress
         self.test_class.output = None
+        self.execution_time = None
 
     def run(self, command, gdb=False, old_binary=False, return_id=True, env=None,
             skip_log_directory=False, expect_error=False, use_backup_dir=True):
@@ -110,16 +114,17 @@ class ProbackupApp:
                 cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline
                 print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port))
 
+            start_time = time.time()
             self.test_class.output = subprocess.check_output(
                 cmdline,
                 stderr=subprocess.STDOUT,
                 env=env
             ).decode('utf-8', errors='replace')
+            end_time = time.time()
+            self.execution_time = end_time - start_time
+
             if command[0] == 'backup' and return_id:
-                # return backup ID
-                for line in self.test_class.output.splitlines():
-                    if 'INFO: Backup' and 'completed' in line:
-                        result = line.split()[2]
+                result = self.get_backup_id()
             else:
                 result = self.test_class.output
             if expect_error is True:
@@ -136,6 +141,19 @@ class ProbackupApp:
             else:
                 raise ProbackupException(self.test_class.output, self.test_class.cmd)
 
+    def get_backup_id(self):
+        if init_params.major_version > 2:
+            pattern = re.compile(r"Backup (.*) completed successfully.")
+            for line in self.test_class.output.splitlines():
+                match = pattern.search(line)
+                if match:
+                    return match.group(1)
+        else:
+            for line in self.test_class.output.splitlines():
+                if 'INFO: Backup' and 'completed' in line:
+                    return line.split()[2]
+        return None
+
     def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True):
         if options is None:
             options = []
@@ -146,13 +164,19 @@ class ProbackupApp:
                         use_backup_dir=use_backup_dir
                         )
 
-    def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False):
+    def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False):
         if options is None:
             options = []
+
+        if not datname:
+            datname = 'postgres'
+
         cmd = [
             'add-instance',
             '--instance={0}'.format(instance),
-            '-D', node.data_dir
+            '-D', node.data_dir,
+            '--pgport', '%i' % node.port,
+            '--pgdatabase', datname
         ]
 
         # don`t forget to kill old_binary after remote ssh release
@@ -189,15 +213,13 @@ class ProbackupApp:
 
         return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error)
 
-    def del_instance(self, instance, old_binary=False, expect_error=False):
-
-        return self.run([
-            'del-instance',
-            '--instance={0}'.format(instance),
-        ],
-            old_binary=old_binary,
-            expect_error=expect_error
-        )
+    def del_instance(self, instance, options=None, old_binary=False, expect_error=False):
+        if options is None:
+            options = []
+        cmd = ['del-instance', '--instance={0}'.format(instance)] + options
+        return self.run(cmd,
+                        old_binary=old_binary,
+                        expect_error=expect_error)
 
     def backup_node(
             self, instance, node, data_dir=False,
@@ -662,7 +684,11 @@ class ProbackupApp:
         if custom_archive_command is None:
             archive_command = " ".join([f'"{init_params.probackup_path}"',
                                         'archive-push', *self.backup_dir.pb_args])
-            if os.name == "nt":
+            if os.name == 'posix':
+                # Dash produces a core dump when it gets a SIGQUIT from its
+                # child process so replace the shell with pg_probackup
+                archive_command = 'exec ' + archive_command
+            elif os.name == "nt":
                 archive_command = archive_command.replace("\\", "\\\\")
             archive_command += f' --instance={instance}'
 
@@ -747,16 +773,30 @@ class ProbackupApp:
 
         return getattr(module, class_name)
 
+    def archive_push(self, instance, node, wal_file_name, wal_file_path=None, options=None, expect_error=False):
+        if options is None:
+            options = []
+        cmd = [
+            'archive-push',
+            '--instance={0}'.format(instance),
+            '--wal-file-name={0}'.format(wal_file_name),
+        ]
+        if wal_file_path is None:
+            cmd = cmd + ['--wal-file-path={0}'.format(os.path.join(node.data_dir, 'pg_wal'))]
+        else:
+            cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)]
+        return self.run(cmd + options, expect_error=expect_error)
 
-# Local or S3 backup
-fs_backup_class = FSTestBackupDir
-if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')):
-    root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
-    if root not in sys.path:
-        sys.path.append(root)
-    from pg_probackup2.storage.s3_backup import S3TestBackupDir
-
-    fs_backup_class = S3TestBackupDir
+    def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expect_error=False):
+        if options is None:
+            options = []
+        cmd = [
+            'archive-get',
+            '--instance={0}'.format(instance),
+            '--wal-file-name={0}'.format(wal_file_name),
+            '--wal-file-path={0}'.format(wal_file_path),
+        ]
+        return self.run(cmd + options, expect_error=expect_error)
 
     def build_backup_dir(self, backup='backup'):
         return fs_backup_class(rel_path=self.rel_path, backup=backup)

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/pg_probackup2/init_helpers.py

@@ -52,6 +52,7 @@ class Init(object):
         parts[0] = re.match(r'\d+', parts[0]).group()
         self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0)
 
+        os.environ['LANGUAGE'] = 'en'   # set default locale language to en. All messages will use this locale
         test_env = os.environ.copy()
         envs_list = [
             'LANGUAGE',
@@ -147,21 +148,24 @@ class Init(object):
             [self.probackup_path, "--version"],
             stderr=subprocess.STDOUT,
         ).decode('utf-8')
-        self.probackup_version = re.search(r"\d+\.\d+\.\d+",
-                                           probackup_version_output
-                                           ).group(0)
-        compressions = re.search(r"\(compressions: ([^)]*)\)",
-                                 probackup_version_output).group(1)
-        self.probackup_compressions = {s.strip() for s in compressions.split(',')}
+        match = re.search(r"\d+\.\d+\.\d+",
+                          probackup_version_output)
+        self.probackup_version = match.group(0) if match else None
+        match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output)
+        compressions = match.group(1) if match else None
+        if compressions:
+            self.probackup_compressions = {s.strip() for s in compressions.split(',')}
+        else:
+            self.probackup_compressions = []
 
         if self.probackup_old_path:
             old_probackup_version_output = subprocess.check_output(
                 [self.probackup_old_path, "--version"],
                 stderr=subprocess.STDOUT,
             ).decode('utf-8')
-            self.old_probackup_version = re.search(r"\d+\.\d+\.\d+",
-                                                   old_probackup_version_output
-                                                   ).group(0)
+            match = re.search(r"\d+\.\d+\.\d+",
+                              old_probackup_version_output)
+            self.old_probackup_version = match.group(0) if match else None
 
         self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON'
         self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000
@@ -200,6 +204,12 @@ class Init(object):
         os.environ["PGAPPNAME"] = "pg_probackup"
         self.delete_logs = delete_logs
 
+        if self.probackup_version.split('.')[0].isdigit():
+            self.major_version = int(self.probackup_version.split('.')[0])
+        else:
+            print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version))
+            sys.exit(1)
+
     def test_env(self):
         return self._test_env.copy()
 

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/pg_probackup2/storage/fs_backup.py

@@ -43,7 +43,10 @@ class FSTestBackupDir(TestBackupDir):
     """ Backup directory. Usually created by running pg_probackup init -B <path>"""
 
     def __init__(self, *, rel_path, backup):
-        self.path = os.path.join(init_params.tmp_path, rel_path, backup)
+        backup_prefix = os.environ.get('PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX')
+        if backup_prefix and not os.path.isabs(backup_prefix):
+            raise Exception(f"PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX must be an absolute path, current value: {backup_prefix}")
+        self.path = os.path.join(backup_prefix or init_params.tmp_path, rel_path, backup)
         self.pb_args = ('-B', self.path)
 
     def list_instance_backups(self, instance):

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/setup.py

@@ -4,7 +4,7 @@ except ImportError:
     from distutils.core import setup
 
 setup(
-    version='0.0.1',
+    version='0.0.3',
     name='testgres_pg_probackup2',
     packages=['pg_probackup2', 'pg_probackup2.storage'],
     description='Plugin for testgres that manages pg_probackup2',

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/testgres_pg_probackup2.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: testgres-pg-probackup2
-Version: 0.0.1
+Version: 0.0.3
 Summary: Plugin for testgres that manages pg_probackup2
 Home-page: https://github.com/postgrespro/testgres
 Author: Postgres Professional

{testgres_pg_probackup2-0.0.1 → testgres_pg_probackup2-0.0.3}/testgres_pg_probackup2.egg-info/SOURCES.txt

@@ -6,7 +6,6 @@ pg_probackup2/gdb.py
 pg_probackup2/init_helpers.py
 pg_probackup2/storage/__init__.py
 pg_probackup2/storage/fs_backup.py
-pg_probackup2/storage/s3_backup.py
 testgres_pg_probackup2.egg-info/PKG-INFO
 testgres_pg_probackup2.egg-info/SOURCES.txt
 testgres_pg_probackup2.egg-info/dependency_links.txt

testgres_pg_probackup2-0.0.1/pg_probackup2/storage/s3_backup.py

@@ -1,134 +0,0 @@
-import os
-import io
-import sys
-
-import minio
-from minio import Minio
-from minio.deleteobjects import DeleteObject
-import urllib3
-from .fs_backup import TestBackupDir
-
-root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..'))
-if root not in sys.path:
-    sys.path.append(root)
-
-# Should fail if either of env vars does not exist
-host = os.environ['PG_PROBACKUP_S3_HOST']
-port = os.environ['PG_PROBACKUP_S3_PORT']
-access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY']
-secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY']
-bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME']
-path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX")
-https = os.environ.get("PG_PROBACKUP_S3_HTTPS")
-
-s3_type = os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST'))
-tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='')
-
-status_forcelist = [413,  # RequestBodyTooLarge
-                    429,  # TooManyRequests
-                    500,  # InternalError
-                    503,  # ServerBusy
-                    ]
-
-
-class S3TestBackupDir(TestBackupDir):
-    is_file_based = False
-
-    def __init__(self, *, rel_path, backup):
-        path = "pg_probackup"
-        if path_suffix:
-            path += "_" + path_suffix
-        if tmp_path == '' or os.path.isabs(tmp_path):
-            self.path = f"{path}{tmp_path}/{rel_path}/{backup}"
-        else:
-            self.path = f"{path}/{tmp_path}/{rel_path}/{backup}"
-
-        secure: bool = False
-        if https in ['ON', 'HTTPS']:
-            secure = True
-        self.conn = Minio(host + ":" + port, secure=secure, access_key=access,
-                          secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5,
-                                                                                                   backoff_factor=1,
-                                                                                                   status_forcelist=status_forcelist)))
-        if not self.conn.bucket_exists(bucket):
-            raise Exception(f"Test bucket {bucket} does not exist.")
-        self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}')
-        return
-
-    def list_instance_backups(self, instance):
-        full_path = os.path.join(self.path, 'backups', instance)
-        candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True)
-        return [os.path.basename(os.path.dirname(x.object_name))
-                for x in candidates if x.object_name.endswith('backup.control')]
-
-    def list_files(self, sub_dir, recursive=False):
-        full_path = os.path.join(self.path, sub_dir)
-        #  Need '/' in the end to find inside the folder
-        full_path_dir = full_path if full_path[-1] == '/' else full_path + '/'
-        object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive)
-        return [obj.object_name.replace(full_path_dir, '', 1)
-                for obj in object_list
-                if not obj.is_dir]
-
-    def list_dirs(self, sub_dir):
-        full_path = os.path.join(self.path, sub_dir)
-        #  Need '/' in the end to find inside the folder
-        full_path_dir = full_path if full_path[-1] == '/' else full_path + '/'
-        object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=False)
-        return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/')
-                for obj in object_list
-                if obj.is_dir]
-
-    def read_file(self, sub_path, *, text=True):
-        full_path = os.path.join(self.path, sub_path)
-        bytes = self.conn.get_object(bucket, full_path).read()
-        if not text:
-            return bytes
-        return bytes.decode('utf-8')
-
-    def write_file(self, sub_path, data, *, text=True):
-        full_path = os.path.join(self.path, sub_path)
-        if text:
-            data = data.encode('utf-8')
-        self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data))
-
-    def cleanup(self):
-        self.remove_dir('')
-
-    def remove_file(self, sub_path):
-        full_path = os.path.join(self.path, sub_path)
-        self.conn.remove_object(bucket, full_path)
-
-    def remove_dir(self, sub_path):
-        if sub_path:
-            full_path = os.path.join(self.path, sub_path)
-        else:
-            full_path = self.path
-        objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True,
-                                      include_version=True)
-        delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs)
-        errs = list(self.conn.remove_objects(bucket, delobjs))
-        if errs:
-            strerrs = "; ".join(str(err) for err in errs)
-            raise Exception("There were errors: {0}".format(strerrs))
-
-    def exists(self, sub_path):
-        full_path = os.path.join(self.path, sub_path)
-        try:
-            self.conn.stat_object(bucket, full_path)
-            return True
-        except minio.error.S3Error as s3err:
-            if s3err.code == 'NoSuchKey':
-                return False
-            raise s3err
-        except Exception as err:
-            raise err
-
-    def __str__(self):
-        return '/' + self.path
-
-    def __repr__(self):
-        return "S3TestBackupDir" + str(self.path)
-
-    def __fspath__(self):
-        return self.path