testgres 1.10.3__tar.gz → 1.10.4__tar.gz

{testgres-1.10.3/testgres.egg-info → testgres-1.10.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: testgres
-Version: 1.10.3
+Version: 1.10.4
 Summary: Testing utility for PostgreSQL and its extensions
 Home-page: https://github.com/postgrespro/testgres
 Author: Postgres Professional
@@ -14,6 +14,15 @@ Requires-Dist: port-for>=0.4
 Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 [![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches)
 [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)

{testgres-1.10.3 → testgres-1.10.4}/setup.py

@@ -27,7 +27,7 @@ with open('README.md', 'r') as f:
     readme = f.read()
 
 setup(
-    version='1.10.3',
+    version='1.10.4',
     name='testgres',
     packages=['testgres', 'testgres.operations', 'testgres.helpers'],
     description='Testing utility for PostgreSQL and its extensions',

{testgres-1.10.3 → testgres-1.10.4}/testgres/__init__.py

@@ -23,7 +23,8 @@ from .exceptions import \
     CatchUpException, \
     StartNodeException, \
     InitNodeException, \
-    BackupException
+    BackupException, \
+    InvalidOperationException
 
 from .enums import \
     XLogMethod, \
@@ -60,7 +61,7 @@ __all__ = [
     "NodeBackup", "testgres_config",
     "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config",
     "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
-    "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException",
+    "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
     "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
     "PostgresNode", "NodeApp",
     "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",

{testgres-1.10.3 → testgres-1.10.4}/testgres/exceptions.py

@@ -9,13 +9,14 @@ class TestgresException(Exception):
 
 @six.python_2_unicode_compatible
 class ExecUtilException(TestgresException):
-    def __init__(self, message=None, command=None, exit_code=0, out=None):
+    def __init__(self, message=None, command=None, exit_code=0, out=None, error=None):
         super(ExecUtilException, self).__init__(message)
 
         self.message = message
         self.command = command
         self.exit_code = exit_code
         self.out = out
+        self.error = error
 
     def __str__(self):
         msg = []
@@ -24,13 +25,17 @@ class ExecUtilException(TestgresException):
             msg.append(self.message)
 
         if self.command:
-            msg.append(u'Command: {}'.format(self.command))
+            command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command,
+            msg.append(u'Command: {}'.format(command_s))
 
         if self.exit_code:
             msg.append(u'Exit code: {}'.format(self.exit_code))
 
+        if self.error:
+            msg.append(u'---- Error:\n{}'.format(self.error))
+
         if self.out:
-            msg.append(u'----\n{}'.format(self.out))
+            msg.append(u'---- Out:\n{}'.format(self.out))
 
         return self.convert_and_join(msg)
 
@@ -98,3 +103,7 @@ class InitNodeException(TestgresException):
 
 class BackupException(TestgresException):
     pass
+
+
+class InvalidOperationException(TestgresException):
+    pass

{testgres-1.10.3 → testgres-1.10.4}/testgres/helpers/port_manager.py

@@ -26,10 +26,11 @@ class PortManager:
         if ports is None:
             ports = set(range(1024, 65535))
 
-        if exclude_ports is None:
-            exclude_ports = set()
+        assert type(ports) == set  # noqa: E721
 
-        ports.difference_update(set(exclude_ports))
+        if exclude_ports is not None:
+            assert isinstance(exclude_ports, Iterable)
+            ports.difference_update(exclude_ports)
 
         sampled_ports = random.sample(tuple(ports), min(len(ports), 100))
 

{testgres-1.10.3 → testgres-1.10.4}/testgres/node.py

@@ -5,6 +5,7 @@ import random
 import signal
 import subprocess
 import threading
+import tempfile
 from queue import Queue
 
 import time
@@ -73,7 +74,8 @@ from .exceptions import \
     TimeoutException,   \
     InitNodeException,  \
     TestgresException,  \
-    BackupException
+    BackupException,    \
+    InvalidOperationException
 
 from .logger import TestgresLogger
 
@@ -81,13 +83,13 @@ from .pubsub import Publication, Subscription
 
 from .standby import First
 
+from . import utils
+
 from .utils import \
     PgVer, \
     eprint, \
     get_bin_path, \
     get_pg_version, \
-    reserve_port, \
-    release_port, \
     execute_utility, \
     options_string, \
     clean_on_error
@@ -126,7 +128,11 @@ class ProcessProxy(object):
 
 
 class PostgresNode(object):
-    def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None):
+    # a max number of node start attempts
+    _C_MAX_START_ATEMPTS = 5
+
+    def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(),
+                 bin_dir=None, prefix=None):
         """
         PostgresNode constructor.
 
@@ -156,7 +162,7 @@ class PostgresNode(object):
             self.os_ops = LocalOperations(conn_params)
 
         self.host = self.os_ops.host
-        self.port = port or reserve_port()
+        self.port = port or utils.reserve_port()
 
         self.ssh_key = self.os_ops.ssh_key
 
@@ -469,6 +475,28 @@ class PostgresNode(object):
 
         return result
 
+    def _collect_log_files(self):
+        # dictionary of log files + size in bytes
+
+        files = [
+            self.pg_log_file
+        ]  # yapf: disable
+
+        result = {}
+
+        for f in files:
+            # skip missing files
+            if not self.os_ops.path_exists(f):
+                continue
+
+            file_size = self.os_ops.get_file_size(f)
+            assert type(file_size) == int  # noqa: E721
+            assert file_size >= 0
+
+            result[f] = file_size
+
+        return result
+
     def init(self, initdb_params=None, cached=True, **kwargs):
         """
         Perform initdb for this node.
@@ -720,6 +748,22 @@ class PostgresNode(object):
                                         OperationalError},
                               max_attempts=max_attempts)
 
+    def _detect_port_conflict(self, log_files0, log_files1):
+        assert type(log_files0) == dict  # noqa: E721
+        assert type(log_files1) == dict  # noqa: E721
+
+        for file in log_files1.keys():
+            read_pos = 0
+
+            if file in log_files0.keys():
+                read_pos = log_files0[file]  # the previous size
+
+            file_content = self.os_ops.read_binary(file, read_pos)
+            file_content_s = file_content.decode()
+            if 'Is another postmaster already running on port' in file_content_s:
+                return True
+        return False
+
     def start(self, params=[], wait=True):
         """
         Starts the PostgreSQL node using pg_ctl if node has not been started.
@@ -734,40 +778,85 @@ class PostgresNode(object):
         Returns:
             This instance of :class:`.PostgresNode`.
         """
+
+        assert __class__._C_MAX_START_ATEMPTS > 1
+
         if self.is_started:
             return self
 
-        _params = [
-            self._get_bin_path("pg_ctl"),
-            "-D", self.data_dir,
-            "-l", self.pg_log_file,
-            "-w" if wait else '-W',  # --wait or --no-wait
-            "start"
-        ] + params  # yapf: disable
+        _params = [self._get_bin_path("pg_ctl"),
+                   "-D", self.data_dir,
+                   "-l", self.pg_log_file,
+                   "-w" if wait else '-W',  # --wait or --no-wait
+                   "start"] + params  # yapf: disable
 
-        startup_retries = 5
-        while True:
+        def LOCAL__start_node():
+            # 'error' will be None on Windows
+            _, _, error = execute_utility(_params, self.utils_log_file, verbose=True)
+            assert error is None or type(error) == str  # noqa: E721
+            if error and 'does not exist' in error:
+                raise Exception(error)
+
+        def LOCAL__raise_cannot_start_node(from_exception, msg):
+            assert isinstance(from_exception, Exception)
+            assert type(msg) == str  # noqa: E721
+            files = self._collect_special_files()
+            raise_from(StartNodeException(msg, files), from_exception)
+
+        def LOCAL__raise_cannot_start_node__std(from_exception):
+            assert isinstance(from_exception, Exception)
+            LOCAL__raise_cannot_start_node(from_exception, 'Cannot start node')
+
+        if not self._should_free_port:
             try:
-                exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True)
-                if error and 'does not exist' in error:
-                    raise Exception
+                LOCAL__start_node()
             except Exception as e:
-                files = self._collect_special_files()
-                if any(len(file) > 1 and 'Is another postmaster already '
-                                         'running on port' in file[1].decode() for
-                       file in files):
-                    logging.warning("Detected an issue with connecting to port {0}. "
-                                    "Trying another port after a 5-second sleep...".format(self.port))
-                    self.port = reserve_port()
-                    options = {'port': str(self.port)}
-                    self.set_auto_conf(options)
-                    startup_retries -= 1
-                    time.sleep(5)
+                LOCAL__raise_cannot_start_node__std(e)
+        else:
+            assert self._should_free_port
+            assert __class__._C_MAX_START_ATEMPTS > 1
+
+            log_files0 = self._collect_log_files()
+            assert type(log_files0) == dict  # noqa: E721
+
+            nAttempt = 0
+            timeout = 1
+            while True:
+                assert nAttempt >= 0
+                assert nAttempt < __class__._C_MAX_START_ATEMPTS
+                nAttempt += 1
+                try:
+                    LOCAL__start_node()
+                except Exception as e:
+                    assert nAttempt > 0
+                    assert nAttempt <= __class__._C_MAX_START_ATEMPTS
+                    if nAttempt == __class__._C_MAX_START_ATEMPTS:
+                        logging.error("Reached maximum retry attempts. Unable to start node.")
+                        LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts")
+
+                    log_files1 = self._collect_log_files()
+                    if not self._detect_port_conflict(log_files0, log_files1):
+                        LOCAL__raise_cannot_start_node__std(e)
+
+                    log_files0 = log_files1
+                    logging.warning(
+                        "Detected a conflict with using the port {0}. "
+                        "Trying another port after a {1}-second sleep...".format(self.port, timeout)
+                    )
+                    time.sleep(timeout)
+                    timeout = min(2 * timeout, 5)
+                    cur_port = self.port
+                    new_port = utils.reserve_port()  # can raise
+                    try:
+                        options = {'port': new_port}
+                        self.set_auto_conf(options)
+                    except:  # noqa: E722
+                        utils.release_port(new_port)
+                        raise
+                    self.port = new_port
+                    utils.release_port(cur_port)
                     continue
-
-                msg = 'Cannot start node'
-                raise_from(StartNodeException(msg, files), e)
-            break
+                break
         self._maybe_start_logger()
         self.is_started = True
         return self
@@ -930,8 +1019,10 @@ class PostgresNode(object):
         """
 
         if self._should_free_port:
+            port = self.port
             self._should_free_port = False
-            release_port(self.port)
+            self.port = None
+            utils.release_port(port)
 
     def cleanup(self, max_attempts=3, full=False):
         """
@@ -986,13 +1077,43 @@ class PostgresNode(object):
             >>> psql(query='select 3', ON_ERROR_STOP=1)
         """
 
-        dbname = dbname or default_dbname()
+        return self._psql(
+            ignore_errors=True,
+            query=query,
+            filename=filename,
+            dbname=dbname,
+            username=username,
+            input=input,
+            **variables
+        )
+
+    def _psql(
+            self,
+            ignore_errors,
+            query=None,
+            filename=None,
+            dbname=None,
+            username=None,
+            input=None,
+            **variables):
+        assert type(variables) == dict  # noqa: E721
+
+        #
+        # We do not support encoding. It may be added later. Ok?
+        #
+        if input is None:
+            pass
+        elif type(input) == bytes:  # noqa: E721
+            pass
+        else:
+            raise Exception("Input data must be None or bytes.")
 
         psql_params = [
             self._get_bin_path("psql"),
             "-p", str(self.port),
             "-h", self.host,
             "-U", username or self.os_ops.username,
+            "-d", dbname or default_dbname(),
             "-X",  # no .psqlrc
             "-A",  # unaligned output
             "-t",  # print rows only
@@ -1005,31 +1126,19 @@ class PostgresNode(object):
 
         # select query source
         if query:
-            if self.os_ops.remote:
-                psql_params.extend(("-c", '"{}"'.format(query)))
-            else:
-                psql_params.extend(("-c", query))
+            psql_params.extend(("-c", query))
         elif filename:
             psql_params.extend(("-f", filename))
         else:
             raise QueryException('Query or filename must be provided')
 
-        # should be the last one
-        psql_params.append(dbname)
-        if not self.os_ops.remote:
-            # start psql process
-            process = subprocess.Popen(psql_params,
-                                       stdin=subprocess.PIPE,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-
-            # wait until it finishes and get stdout and stderr
-            out, err = process.communicate(input=input)
-            return process.returncode, out, err
-        else:
-            status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)
-
-            return status_code, out, err
+        return self.os_ops.exec_command(
+            psql_params,
+            verbose=True,
+            input=input,
+            stderr=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            ignore_errors=ignore_errors)
 
     @method_decorator(positional_args_hack(['dbname', 'query']))
     def safe_psql(self, query=None, expect_error=False, **kwargs):
@@ -1050,22 +1159,27 @@ class PostgresNode(object):
         Returns:
             psql's output as str.
         """
+        assert type(kwargs) == dict  # noqa: E721
+        assert not ("ignore_errors" in kwargs.keys())
+        assert not ("expect_error" in kwargs.keys())
 
         # force this setting
         kwargs['ON_ERROR_STOP'] = 1
         try:
-            ret, out, err = self.psql(query=query, **kwargs)
+            ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs)
         except ExecUtilException as e:
-            ret = e.exit_code
-            out = e.out
-            err = e.message
-        if ret:
-            if expect_error:
-                out = (err or b'').decode('utf-8')
-            else:
-                raise QueryException((err or b'').decode('utf-8'), query)
-        elif expect_error:
-            assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)
+            if not expect_error:
+                raise QueryException(e.message, query)
+
+            if type(e.error) == bytes:  # noqa: E721
+                return e.error.decode("utf-8")  # throw
+
+            # [2024-12-09] This situation is not expected
+            assert False
+            return e.error
+
+        if expect_error:
+            raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))
 
         return out
 
@@ -1626,23 +1740,31 @@ class PostgresNode(object):
 
             name, var = line.partition('=')[::2]
             name = name.strip()
-            var = var.strip()
-            var = var.strip('"')
-            var = var.strip("'")
 
-            # remove options specified in rm_options list
+            # Remove options specified in rm_options list
             if name in rm_options:
                 continue
 
             current_options[name] = var
 
         for option in options:
-            current_options[option] = options[option]
+            assert type(option) == str  # noqa: E721
+            assert option != ""
+            assert option.strip() == option
+
+            value = options[option]
+            valueType = type(value)
+
+            if valueType == str:
+                value = __class__._escape_config_value(value)
+            elif valueType == bool:
+                value = "on" if value else "off"
+
+            current_options[option] = value
 
         auto_conf = ''
         for option in current_options:
-            auto_conf += "{0} = '{1}'\n".format(
-                option, current_options[option])
+            auto_conf += option + " = " + str(current_options[option]) + "\n"
 
         for directive in current_directives:
             auto_conf += directive + "\n"
@@ -1690,6 +1812,30 @@ class PostgresNode(object):
             bin_path = get_bin_path(filename)
         return bin_path
 
+    def _escape_config_value(value):
+        assert type(value) == str  # noqa: E721
+
+        result = "'"
+
+        for ch in value:
+            if ch == "'":
+                result += "\\'"
+            elif ch == "\n":
+                result += "\\n"
+            elif ch == "\r":
+                result += "\\r"
+            elif ch == "\t":
+                result += "\\t"
+            elif ch == "\b":
+                result += "\\b"
+            elif ch == "\\":
+                result += "\\\\"
+            else:
+                result += ch
+
+        result += "'"
+        return result
+
 
 class NodeApp:
 
@@ -1729,6 +1875,8 @@ class NodeApp:
             pg_options={},
             checksum=True,
             bin_dir=None):
+        assert type(pg_options) == dict  # noqa: E721
+
         if checksum and '--data-checksums' not in initdb_params:
             initdb_params.append('--data-checksums')
         node = self.make_empty(base_dir, port, bin_dir=bin_dir)
@@ -1741,20 +1889,22 @@ class NodeApp:
         node.major_version = float(node.major_version_str)
 
         # Set default parameters
-        options = {'max_connections': 100,
-                   'shared_buffers': '10MB',
-                   'fsync': 'off',
-                   'wal_level': 'logical',
-                   'hot_standby': 'off',
-                   'log_line_prefix': '%t [%p]: [%l-1] ',
-                   'log_statement': 'none',
-                   'log_duration': 'on',
-                   'log_min_duration_statement': 0,
-                   'log_connections': 'on',
-                   'log_disconnections': 'on',
-                   'restart_after_crash': 'off',
-                   'autovacuum': 'off',
-                   'unix_socket_directories': '/tmp'}
+        options = {
+            'max_connections': 100,
+            'shared_buffers': '10MB',
+            'fsync': 'off',
+            'wal_level': 'logical',
+            'hot_standby': 'off',
+            'log_line_prefix': '%t [%p]: [%l-1] ',
+            'log_statement': 'none',
+            'log_duration': 'on',
+            'log_min_duration_statement': 0,
+            'log_connections': 'on',
+            'log_disconnections': 'on',
+            'restart_after_crash': 'off',
+            'autovacuum': 'off',
+            # unix_socket_directories will be defined later
+        }
 
         # Allow replication in pg_hba.conf
         if set_replication:
@@ -1769,11 +1919,16 @@ class NodeApp:
         else:
             options['wal_keep_segments'] = '12'
 
-        # set default values
-        node.set_auto_conf(options)
-
         # Apply given parameters
-        node.set_auto_conf(pg_options)
+        for option_name, option_value in iteritems(pg_options):
+            options[option_name] = option_value
+
+        # Define delayed propertyes
+        if not ("unix_socket_directories" in options.keys()):
+            options["unix_socket_directories"] = __class__._gettempdir()
+
+        # Set config values
+        node.set_auto_conf(options)
 
         # kludge for testgres
         # https://github.com/postgrespro/testgres/issues/54
@@ -1782,3 +1937,26 @@ class NodeApp:
             node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments'])
 
         return node
+
+    def _gettempdir():
+        v = tempfile.gettempdir()
+
+        #
+        # Paranoid checks
+        #
+        if type(v) != str:  # noqa: E721
+            __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__))
+
+        if v == "":
+            __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.")
+
+        if not os.path.exists(v):
+            __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v))
+
+        # OK
+        return v
+
+    def _raise_bugcheck(msg):
+        assert type(msg) == str  # noqa: E721
+        assert msg != ""
+        raise Exception("[BUG CHECK] " + msg)

testgres-1.10.4/testgres/operations/helpers.py

@@ -0,0 +1,52 @@
+import locale
+
+
+class Helpers:
+    def _make_get_default_encoding_func():
+        # locale.getencoding is added in Python 3.11
+        if hasattr(locale, 'getencoding'):
+            return locale.getencoding
+
+        # It must exist
+        return locale.getpreferredencoding
+
+    # Prepared pointer on function to get a name of system codepage
+    _get_default_encoding_func = _make_get_default_encoding_func()
+
+    def GetDefaultEncoding():
+        #
+        #   Original idea/source was:
+        #
+        #   def os_ops.get_default_encoding():
+        #       if not hasattr(locale, 'getencoding'):
+        #       locale.getencoding = locale.getpreferredencoding
+        #       return locale.getencoding() or 'UTF-8'
+        #
+
+        assert __class__._get_default_encoding_func is not None
+
+        r = __class__._get_default_encoding_func()
+
+        if r:
+            assert r is not None
+            assert type(r) == str  # noqa: E721
+            assert r != ""
+            return r
+
+        # Is it an unexpected situation?
+        return 'UTF-8'
+
+    def PrepareProcessInput(input, encoding):
+        if not input:
+            return None
+
+        if type(input) == str:  # noqa: E721
+            if encoding is None:
+                return input.encode(__class__.GetDefaultEncoding())
+
+            assert type(encoding) == str  # noqa: E721
+            return input.encode(encoding)
+
+        # It is expected!
+        assert type(input) == bytes  # noqa: E721
+        return input