testgres 1.9.3__tar.gz → 1.10.1__tar.gz

{testgres-1.9.3/testgres.egg-info → testgres-1.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: testgres
-Version: 1.9.3
+Version: 1.10.1
 Summary: Testing utility for PostgreSQL and its extensions
 Home-page: https://github.com/postgrespro/testgres
 Author: Postgres Professional

{testgres-1.9.3 → testgres-1.10.1}/setup.py

@@ -27,7 +27,7 @@ with open('README.md', 'r') as f:
     readme = f.read()
 
 setup(
-    version='1.9.3',
+    version='1.10.1',
     name='testgres',
     packages=['testgres', 'testgres.operations', 'testgres.helpers'],
     description='Testing utility for PostgreSQL and its extensions',

{testgres-1.9.3 → testgres-1.10.1}/testgres/backup.py

@@ -33,7 +33,8 @@ class NodeBackup(object):
                  node,
                  base_dir=None,
                  username=None,
-                 xlog_method=XLogMethod.fetch):
+                 xlog_method=XLogMethod.fetch,
+                 options=None):
         """
         Create a new backup.
 
@@ -43,6 +44,8 @@ class NodeBackup(object):
             username: database user name.
             xlog_method: none | fetch | stream (see docs)
         """
+        if not options:
+            options = []
         self.os_ops = node.os_ops
         if not node.status():
             raise BackupException('Node must be running')
@@ -77,6 +80,7 @@ class NodeBackup(object):
             "-D", data_dir,
             "-X", xlog_method.value
         ]  # yapf: disable
+        _params += options
         execute_utility(_params, self.log_file)
 
     def __enter__(self):

{testgres-1.9.3 → testgres-1.10.1}/testgres/cache.py

@@ -22,19 +22,20 @@ from .operations.local_ops import LocalOperations
 from .operations.os_ops import OsOperations
 
 
-def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations()):
+def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True):
     """
     Perform initdb or use cached node files.
     """
 
     def call_initdb(initdb_dir, log=logfile):
         try:
-            _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"]
+            initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb")
+            _params = [initdb_path, "-D", initdb_dir, "-N"]
             execute_utility(_params + (params or []), log)
         except ExecUtilException as e:
             raise_from(InitNodeException("Failed to run initdb"), e)
 
-    if params or not testgres_config.cache_initdb:
+    if params or not testgres_config.cache_initdb or not cached:
         call_initdb(data_dir, logfile)
     else:
         # Fetch cached initdb dir

{testgres-1.9.3 → testgres-1.10.1}/testgres/connection.py

@@ -104,14 +104,13 @@ class NodeConnection(object):
     def execute(self, query, *args):
         self.cursor.execute(query, args)
         try:
-            res = self.cursor.fetchall()
             # pg8000 might return tuples
-            if isinstance(res, tuple):
-                res = [tuple(t) for t in res]
-
+            res = [tuple(t) for t in self.cursor.fetchall()]
             return res
+        except ProgrammingError:
+            return None
         except Exception as e:
-            print("Error executing query: {}".format(e))
+            print("Error executing query: {}\n {}".format(repr(e), query))
             return None
 
     def close(self):

{testgres-1.9.3 → testgres-1.10.1}/testgres/node.py

@@ -127,7 +127,7 @@ class ProcessProxy(object):
 
 
 class PostgresNode(object):
-    def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams()):
+    def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None):
         """
         PostgresNode constructor.
 
@@ -135,12 +135,15 @@ class PostgresNode(object):
             name: node's application name.
             port: port to accept connections.
             base_dir: path to node's data directory.
+            bin_dir: path to node's binary directory.
         """
 
         # private
-        self._pg_version = PgVer(get_pg_version())
+        self._pg_version = PgVer(get_pg_version(bin_dir))
         self._should_free_port = port is None
         self._base_dir = base_dir
+        self._bin_dir = bin_dir
+        self._prefix = prefix
         self._logger = None
         self._master = None
 
@@ -281,7 +284,7 @@ class PostgresNode(object):
     @property
     def base_dir(self):
         if not self._base_dir:
-            self._base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE)
+            self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE)
 
         # NOTE: it's safe to create a new dir
         if not self.os_ops.path_exists(self._base_dir):
@@ -289,6 +292,12 @@ class PostgresNode(object):
 
         return self._base_dir
 
+    @property
+    def bin_dir(self):
+        if not self._bin_dir:
+            self._bin_dir = os.path.dirname(get_bin_path("pg_config"))
+        return self._bin_dir
+
     @property
     def logs_dir(self):
         path = os.path.join(self.base_dir, LOGS_DIR)
@@ -441,7 +450,7 @@ class PostgresNode(object):
 
         return result
 
-    def init(self, initdb_params=None, **kwargs):
+    def init(self, initdb_params=None, cached=True, **kwargs):
         """
         Perform initdb for this node.
 
@@ -460,7 +469,9 @@ class PostgresNode(object):
             data_dir=self.data_dir,
             logfile=self.utils_log_file,
             os_ops=self.os_ops,
-            params=initdb_params)
+            params=initdb_params,
+            bin_path=self.bin_dir,
+            cached=False)
 
         # initialize default config files
         self.default_conf(**kwargs)
@@ -619,7 +630,7 @@ class PostgresNode(object):
 
         try:
             _params = [
-                get_bin_path("pg_ctl"),
+                self._get_bin_path('pg_ctl'),
                 "-D", self.data_dir,
                 "status"
             ]  # yapf: disable
@@ -645,7 +656,7 @@ class PostgresNode(object):
         """
 
         # this one is tricky (blame PG 9.4)
-        _params = [get_bin_path("pg_controldata")]
+        _params = [self._get_bin_path("pg_controldata")]
         _params += ["-D"] if self._pg_version >= PgVer('9.5') else []
         _params += [self.data_dir]
 
@@ -708,21 +719,37 @@ class PostgresNode(object):
             return self
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "-l", self.pg_log_file,
             "-w" if wait else '-W',  # --wait or --no-wait
             "start"
         ] + params  # yapf: disable
 
-        try:
-            exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True)
-            if error and 'does not exist' in error:
-                raise Exception
-        except Exception as e:
-            msg = 'Cannot start node'
-            files = self._collect_special_files()
-            raise_from(StartNodeException(msg, files), e)
+        startup_retries = 5
+        while True:
+            try:
+                exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True)
+                if error and 'does not exist' in error:
+                    raise Exception
+            except Exception as e:
+                files = self._collect_special_files()
+                if any(len(file) > 1 and 'Is another postmaster already '
+                                         'running on port' in file[1].decode() for
+                       file in files):
+                    print("Detected an issue with connecting to port {0}. "
+                          "Trying another port after a 5-second sleep...".format(self.port))
+                    self.port = reserve_port()
+                    options = {}
+                    options['port'] = str(self.port)
+                    self.set_auto_conf(options)
+                    startup_retries -= 1
+                    time.sleep(5)
+                    continue
+
+                msg = 'Cannot start node'
+                raise_from(StartNodeException(msg, files), e)
+            break
         self._maybe_start_logger()
         self.is_started = True
         return self
@@ -742,7 +769,7 @@ class PostgresNode(object):
             return self
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "-w" if wait else '-W',  # --wait or --no-wait
             "stop"
@@ -782,7 +809,7 @@ class PostgresNode(object):
         """
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "-l", self.pg_log_file,
             "-w",  # wait
@@ -814,7 +841,7 @@ class PostgresNode(object):
         """
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "reload"
         ] + params  # yapf: disable
@@ -835,7 +862,7 @@ class PostgresNode(object):
         """
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "-w",  # wait
             "promote"
@@ -871,7 +898,7 @@ class PostgresNode(object):
         """
 
         _params = [
-            get_bin_path("pg_ctl"),
+            self._get_bin_path("pg_ctl"),
             "-D", self.data_dir,
             "-w"  # wait
         ] + params  # yapf: disable
@@ -945,7 +972,7 @@ class PostgresNode(object):
         username = username or default_username()
 
         psql_params = [
-            get_bin_path("psql"),
+            self._get_bin_path("psql"),
             "-p", str(self.port),
             "-h", self.host,
             "-U", username,
@@ -1066,7 +1093,7 @@ class PostgresNode(object):
         filename = filename or tmpfile()
 
         _params = [
-            get_bin_path("pg_dump"),
+            self._get_bin_path("pg_dump"),
             "-p", str(self.port),
             "-h", self.host,
             "-f", filename,
@@ -1094,7 +1121,7 @@ class PostgresNode(object):
         username = username or default_username()
 
         _params = [
-            get_bin_path("pg_restore"),
+            self._get_bin_path("pg_restore"),
             "-p", str(self.port),
             "-h", self.host,
             "-U", username,
@@ -1344,7 +1371,7 @@ class PostgresNode(object):
                 username=None,
                 stdout=None,
                 stderr=None,
-                options=[]):
+                options=None):
         """
         Spawn a pgbench process.
 
@@ -1358,13 +1385,15 @@ class PostgresNode(object):
         Returns:
             Process created by subprocess.Popen.
         """
+        if options is None:
+            options = []
 
         # Set default arguments
         dbname = dbname or default_dbname()
         username = username or default_username()
 
         _params = [
-            get_bin_path("pgbench"),
+            self._get_bin_path("pgbench"),
             "-p", str(self.port),
             "-h", self.host,
             "-U", username,
@@ -1377,6 +1406,29 @@ class PostgresNode(object):
 
         return proc
 
+    def pgbench_with_wait(self,
+                          dbname=None,
+                          username=None,
+                          stdout=None,
+                          stderr=None,
+                          options=None):
+        """
+        Do pgbench command and wait.
+
+        Args:
+            dbname: database name to connect to.
+            username: database user name.
+            stdout: stdout file to be used by Popen.
+            stderr: stderr file to be used by Popen.
+            options: additional options for pgbench (list).
+        """
+        if options is None:
+            options = []
+
+        with self.pgbench(dbname, username, stdout, stderr, options) as pgbench:
+            pgbench.wait()
+        return
+
     def pgbench_init(self, **kwargs):
         """
         Small wrapper for pgbench_run().
@@ -1416,7 +1468,7 @@ class PostgresNode(object):
         username = username or default_username()
 
         _params = [
-            get_bin_path("pgbench"),
+            self._get_bin_path("pgbench"),
             "-p", str(self.port),
             "-h", self.host,
             "-U", username,
@@ -1587,6 +1639,47 @@ class PostgresNode(object):
 
         self.os_ops.write(path, auto_conf, truncate=True)
 
+    def upgrade_from(self, old_node, options=None):
+        """
+        Upgrade this node from an old node using pg_upgrade.
+
+        Args:
+            old_node: An instance of PostgresNode representing the old node.
+        """
+        if not os.path.exists(old_node.data_dir):
+            raise Exception("Old node must be initialized")
+
+        if not os.path.exists(self.data_dir):
+            self.init()
+
+        if not options:
+            options = []
+
+        pg_upgrade_binary = self._get_bin_path("pg_upgrade")
+
+        if not os.path.exists(pg_upgrade_binary):
+            raise Exception("pg_upgrade does not exist in the new node's binary path")
+
+        upgrade_command = [
+            pg_upgrade_binary,
+            "--old-bindir", old_node.bin_dir,
+            "--new-bindir", self.bin_dir,
+            "--old-datadir", old_node.data_dir,
+            "--new-datadir", self.data_dir,
+            "--old-port", str(old_node.port),
+            "--new-port", str(self.port),
+        ]
+        upgrade_command += options
+
+        return self.os_ops.exec_command(upgrade_command)
+
+    def _get_bin_path(self, filename):
+        if self.bin_dir:
+            bin_path = os.path.join(self.bin_dir, filename)
+        else:
+            bin_path = get_bin_path(filename)
+        return bin_path
+
 
 class NodeApp:
 

{testgres-1.9.3 → testgres-1.10.1}/testgres/operations/local_ops.py

@@ -44,7 +44,7 @@ class LocalOperations(OsOperations):
     def _raise_exec_exception(message, command, exit_code, output):
         """Raise an ExecUtilException."""
         raise ExecUtilException(message=message.format(output),
-                                command=command,
+                                command=' '.join(command) if isinstance(command, list) else command,
                                 exit_code=exit_code,
                                 out=output)
 

testgres-1.10.1/testgres/plugins/__init__.py

@@ -0,0 +1,8 @@
+from pg_probackup2.gdb import GDBobj
+from pg_probackup2.app import ProbackupApp, ProbackupException
+from pg_probackup2.init_helpers import init_params
+from pg_probackup2.storage.fs_backup import FSTestBackupDir
+
+__all__ = [
+    "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj"
+]