testgres 1.15.1__tar.gz → 1.16.0__tar.gz

{testgres-1.15.1/testgres.egg-info → testgres-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testgres
-Version: 1.15.1
+Version: 1.16.0
 Summary: Testing utility for PostgreSQL and its extensions
 Author-email: Postgres Professional <testgres@postgrespro.ru>
 License: PostgreSQL
@@ -27,7 +27,7 @@ Requires-Dist: port-for>=0.4
 Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
-Requires-Dist: testgres.os_ops<4.0.0,>=3.2.0
+Requires-Dist: testgres.os_ops<4.0.0,>=3.3.0
 Dynamic: license-file
 
 [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/ci.yml)

{testgres-1.15.1 → testgres-1.16.0}/pyproject.toml

@@ -65,7 +65,7 @@ dependencies = [
     "six>=1.9.0",
     "psutil",
     "packaging",
-    "testgres.os_ops>=3.2.0,<4.0.0",
+    "testgres.os_ops>=3.3.0,<4.0.0",
 ]
 
 [project.urls]

{testgres-1.15.1 → testgres-1.16.0}/src/__init__.py

@@ -18,6 +18,7 @@ from .connection import \
 from .exceptions import \
     TestgresException, \
     ExecUtilException, \
+    ExecTimeoutException, \
     QueryException, \
     QueryTimeoutException, \
     TimeoutException, \
@@ -53,11 +54,13 @@ from .standby import \
 
 from .config import testgres_config
 
-from testgres.operations.os_ops import OsOperations, ConnectionParams
+from testgres.operations.os_ops import OsOperations
+from testgres.operations.os_ops import ConnectionParams
+from testgres.operations.os_ops import OsProcessController
 from testgres.operations.local_ops import LocalOperations
 from testgres.operations.remote_ops import RemoteOperations
 
-__version__ = "1.15.1"
+__version__ = "1.16.0"
 
 __all__ = [
     "get_new_node",
@@ -66,6 +69,7 @@ __all__ = [
     "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config",
     "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
     "TestgresException", "ExecUtilException", "QueryException",
+    "ExecTimeoutException",
     "QueryTimeoutException",
     "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
     "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
@@ -74,5 +78,6 @@ __all__ = [
     "PortManager",
     "reserve_port", "release_port", "get_bin_path", "get_bin_dir", "get_pg_config", "get_pg_version", "parse_pg_version",
     "First", "Any",
-    "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"
+    "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams",
+    "OsProcessController",
 ]

{testgres-1.15.1 → testgres-1.16.0}/src/backup.py

@@ -15,9 +15,10 @@ from .exceptions import BackupException
 
 from testgres.operations.os_ops import OsOperations
 
+from . import utils
+
 from .utils import \
     get_bin_path2, \
-    execute_utility2, \
     clean_on_error
 
 
@@ -86,7 +87,13 @@ class NodeBackup(object):
             "-X", xlog_method.value
         ]  # yapf: disable
         _params += options
-        execute_utility2(self.os_ops, _params, self.log_file)
+
+        utils.execute_utility3(
+            self.os_ops,
+            _params,
+            self.log_file,
+        )
+        return
 
     def __enter__(self):
         return self

{testgres-1.15.1 → testgres-1.16.0}/src/cache.py

@@ -8,14 +8,12 @@ from .consts import XLOG_CONTROL_FILE
 
 from .defaults import generate_system_id
 
+from . import utils
+
 from .exceptions import \
     InitNodeException, \
     ExecUtilException
 
-from .utils import \
-    get_bin_path2, \
-    execute_utility2
-
 from testgres.operations.local_ops import LocalOperations
 from testgres.operations.os_ops import OsOperations
 
@@ -39,13 +37,17 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = No
         if bin_path:
             return os_ops.build_path(bin_path, name)
 
-        return get_bin_path2(os_ops, name)
+        return utils.get_bin_path2(os_ops, name)
 
     def call_initdb(initdb_dir, log=logfile):
         try:
             initdb_path = make_utility_path("initdb")
             _params = [initdb_path, "-D", initdb_dir, "-N"]
-            execute_utility2(os_ops, _params + (params or []), log)
+            utils.execute_utility3(
+                os_ops,
+                _params + (params or []),
+                log,
+            )
         except ExecUtilException as e:
             raise_from(InitNodeException("Failed to run initdb"), e)
 
@@ -78,7 +80,12 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = No
 
                 # XXX: build new WAL segment with our system id
                 _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"]
-                execute_utility2(os_ops, _params, logfile)
+
+                utils.execute_utility3(
+                    os_ops,
+                    _params,
+                    logfile,
+                )
 
         except ExecUtilException as e:
             msg = "Failed to reset WAL for system id"

{testgres-1.15.1 → testgres-1.16.0}/src/exceptions.py

@@ -6,6 +6,7 @@ import typing
 from testgres.operations.exceptions import TestgresException
 from testgres.operations.exceptions import ExecUtilException
 from testgres.operations.exceptions import InvalidOperationException
+from testgres.operations.exceptions import ExecTimeoutException
 
 
 class PortForException(TestgresException):
@@ -298,3 +299,4 @@ class BackupException(TestgresException):
 
 assert ExecUtilException.__name__ == "ExecUtilException"
 assert InvalidOperationException.__name__ == "InvalidOperationException"
+assert ExecTimeoutException.__name__ == "ExecTimeoutException"

{testgres-1.15.1 → testgres-1.16.0}/src/impl/platforms/linux/internal_platform_utils.py

@@ -5,7 +5,9 @@ from ... import internal_utils
 from ....raise_error import RaiseError
 
 from testgres.operations.os_ops import OsOperations
+from testgres.operations.os_ops import OsCommandResult
 from testgres.operations.exceptions import ExecUtilException
+from testgres.operations.types import T_OS_EXEC_ENV
 
 import re
 import shlex
@@ -17,7 +19,7 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
     C_MAX_FIND_POSTMASTER_ATTEMPTS = 5
     C_BASH_EXE = "/bin/bash"
 
-    sm_exec_env = {
+    sm_exec_env: T_OS_EXEC_ENV = {
         "LANG": "en_US.UTF-8",
         "LC_ALL": "en_US.UTF-8",
     }
@@ -113,37 +115,32 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
             "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp),
            ]
 
-        exec_r = os_ops.exec_command(
+        exec_r = os_ops.run(
             cmd=cmd,
-            ignore_errors=True,
-            verbose=True,
+            check=False,
             exec_env=__class__.sm_exec_env,
         )
 
-        assert type(exec_r) is tuple
-        assert len(exec_r) == 3
+        assert type(exec_r) is OsCommandResult
+        assert type(exec_r.returncode) is int
+        assert type(exec_r.stdout) is bytes
+        assert type(exec_r.stderr) is bytes
 
-        exit_status, output_b, error_b = exec_r
-
-        assert type(exit_status) is int
-        assert type(output_b) is bytes
-        assert type(error_b) is bytes
-
-        if exit_status == 1:
+        if exec_r.returncode == 1:
             return None
 
-        output = output_b.decode("utf-8")
-        error = error_b.decode("utf-8")
+        output = exec_r.stdout.decode("utf-8")
+        error = exec_r.stderr.decode("utf-8")
 
         assert type(output) is str
         assert type(error) is str
 
-        if exit_status != 0:
-            errMsg = f"test command returned an unexpected exit code: {exit_status}"
+        if exec_r.returncode != 0:
+            errMsg = f"test command returned an unexpected exit code: {exec_r.returncode}"
             raise ExecUtilException(
                 message=errMsg,
                 command=cmd,
-                exit_code=exit_status,
+                exit_code=exec_r.returncode,
                 out=output,
                 error=error,
             )

{testgres-1.15.1 → testgres-1.16.0}/src/logger.py

@@ -31,7 +31,7 @@ class TestgresLogger(threading.Thread):
         assert type(log_file_encoding) is str
         assert os_ops is None or isinstance(os_ops, OsOperations)
 
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, daemon=True)
 
         if os_ops is None:
             os_ops = tconf.os_ops

{testgres-1.15.1 → testgres-1.16.0}/src/node.py

@@ -92,7 +92,6 @@ from .utils import \
     PgVer, \
     eprint, \
     get_pg_version2, \
-    execute_utility2, \
     options_string, \
     clean_on_error
 
@@ -101,6 +100,8 @@ from .raise_error import RaiseError
 from .backup import NodeBackup
 
 from testgres.operations.os_ops import OsOperations
+from testgres.operations.os_ops import OsCommandResult
+from testgres.operations.os_ops import OsProcessController
 from testgres.operations.local_ops import LocalOperations
 
 InternalError = pglib.InternalError
@@ -638,6 +639,12 @@ class PostgresNode(object):
         assert type(with_force) is bool
         assert max_attempts > 0
 
+        try:
+            self._try_shutdown_internal(max_attempts, with_force)
+        finally:
+            self._maybe_stop_logger()
+
+    def _try_shutdown_internal(self, max_attempts, with_force):
         attempts = 0
 
         # try stopping server N times
@@ -668,7 +675,11 @@ class PostgresNode(object):
 
         ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)]
 
-        ps_output = self._os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')
+        ps_output = self._os_ops.run(
+            cmd=ps_command,
+            shell=True,
+            check=False,
+        ).stdout.decode('utf-8')
         assert type(ps_output) is str
 
         if ps_output == "":
@@ -687,7 +698,11 @@ class PostgresNode(object):
             pass
 
         # Check that node stopped - print only column pid without headers
-        ps_output = self._os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8')
+        ps_output = self._os_ops.run(
+            cmd=ps_command,
+            shell=True,
+            check=False,
+        ).stdout.decode('utf-8')
         assert type(ps_output) is str
 
         if ps_output == "":
@@ -1087,11 +1102,16 @@ class PostgresNode(object):
         _params += ["-D"] if self._pg_version >= PgVer('9.5') else []
         _params += [self.data_dir]
 
-        data = execute_utility2(self._os_ops, _params, self.utils_log_file)
+        exec_r = utils.execute_utility3(
+            self._os_ops,
+            _params,
+            self.utils_log_file,
+        )
+        assert type(exec_r.stdout) is str
 
         out_dict = {}
 
-        for line in data.splitlines():
+        for line in exec_r.stdout.splitlines():
             key, _, value = line.partition(':')
             out_dict[key.strip()] = value.strip()
 
@@ -1242,10 +1262,17 @@ class PostgresNode(object):
 
         def LOCAL__start_node():
             # 'error' will be None on Windows
-            _, _, error = execute_utility2(self._os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env)
-            assert error is None or type(error) is str
-            if error and 'does not exist' in error:
-                raise Exception(error)
+            exec_r = utils.execute_utility3(
+                self._os_ops,
+                _params,
+                self.utils_log_file,
+                exec_env=exec_env,
+            )
+            # TODO: WTF? Remove it!
+            assert exec_r.returncode == 0
+            assert exec_r.stderr is None or type(exec_r.stderr) is str
+            if exec_r.stderr is not None and 'does not exist' in exec_r.stderr:
+                raise ExecUtilException(exec_r.stderr)
 
         def LOCAL__raise_cannot_start_node__std(from_exception):
             assert isinstance(from_exception, Exception)
@@ -1331,11 +1358,17 @@ class PostgresNode(object):
             "stop"
         ] + params  # yapf: disable
 
-        execute_utility2(self._os_ops, _params, self.utils_log_file)
-
-        self._manually_started_pm_pid = None
-
-        self._maybe_stop_logger()
+        try:
+            utils.execute_utility3(
+                self._os_ops,
+                _params,
+                logfile=self.utils_log_file,
+            )
+            self._manually_started_pm_pid = None
+        finally:
+            # always stop the reader thread, even if pg_ctl stop failed,
+            # so it can't leak into interpreter shutdown.
+            self._maybe_stop_logger()
         return self
 
     def kill(self, someone=None):
@@ -1391,9 +1424,16 @@ class PostgresNode(object):
         ] + params  # yapf: disable
 
         try:
-            error_code, out, error = execute_utility2(self._os_ops, _params, self.utils_log_file, verbose=True)
-            if error and 'could not start server' in error:
-                raise ExecUtilException
+            exec_r = utils.execute_utility3(
+                self._os_ops,
+                _params,
+                logfile=self.utils_log_file,
+            )
+            # TODO: WTF? Remove it!
+            assert exec_r.returncode == 0
+            assert exec_r.stderr is None or type(exec_r.stderr) is str
+            if exec_r.stderr is not None and 'could not start server' in exec_r.stderr:
+                raise ExecUtilException(exec_r.stderr)
         except ExecUtilException as e:
             msg = 'Cannot restart node'
             files = self._collect_special_files()
@@ -1420,7 +1460,11 @@ class PostgresNode(object):
             "reload"
         ] + params  # yapf: disable
 
-        execute_utility2(self._os_ops, _params, self.utils_log_file)
+        utils.execute_utility3(
+            self._os_ops,
+            _params,
+            logfile=self.utils_log_file,
+        )
 
         return self
 
@@ -1442,7 +1486,11 @@ class PostgresNode(object):
             "promote"
         ]  # yapf: disable
 
-        execute_utility2(self._os_ops, _params, self.utils_log_file)
+        utils.execute_utility3(
+            self._os_ops,
+            _params,
+            self.utils_log_file,
+        )
 
         # for versions below 10 `promote` is asynchronous so we need to wait
         # until it actually becomes writable
@@ -1477,7 +1525,11 @@ class PostgresNode(object):
             "-w"  # wait
         ] + params  # yapf: disable
 
-        return execute_utility2(self._os_ops, _params, self.utils_log_file)
+        return utils.execute_utility3(
+            self._os_ops,
+            _params,
+            self.utils_log_file,
+        ).stdout
 
     def release_resources(self):
         """
@@ -1556,7 +1608,7 @@ class PostgresNode(object):
         assert port is None or type(port) is int
         assert type(variables) is dict
 
-        return self._psql(
+        r = self._psql(
             ignore_errors=True,
             query=query,
             filename=filename,
@@ -1567,6 +1619,8 @@ class PostgresNode(object):
             port=port,
             **variables
         )
+        assert type(r) is OsCommandResult
+        return r.returncode, r.stdout, r.stderr
 
     def _psql(
             self,
@@ -1578,7 +1632,8 @@ class PostgresNode(object):
             input=None,
             host: typing.Optional[str] = None,
             port: typing.Optional[int] = None,
-            **variables):
+            **variables
+    ) -> OsCommandResult:
         assert host is None or type(host) is str
         assert port is None or type(port) is int
         assert type(variables) is dict
@@ -1628,13 +1683,15 @@ class PostgresNode(object):
         else:
             raise QueryException('Query or filename must be provided')
 
-        return self._os_ops.exec_command(
+        r = self._os_ops.run(
             psql_params,
-            verbose=True,
             input=input,
             stderr=subprocess.PIPE,
             stdout=subprocess.PIPE,
-            ignore_errors=ignore_errors)
+            check=not ignore_errors,
+        )
+        assert type(r) is OsCommandResult
+        return r
 
     @method_decorator(positional_args_hack(['dbname', 'query']))
     def safe_psql(self, query=None, expect_error=False, **kwargs):
@@ -1662,7 +1719,7 @@ class PostgresNode(object):
         # force this setting
         kwargs['ON_ERROR_STOP'] = 1
         try:
-            ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs)
+            exec_r = self._psql(ignore_errors=False, query=query, **kwargs)
         except ExecUtilException as e:
             if not expect_error:
                 raise QueryException(e.message, query)
@@ -1677,7 +1734,7 @@ class PostgresNode(object):
         if expect_error:
             raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))
 
-        return out
+        return exec_r.stdout
 
     def dump(self,
              filename=None,
@@ -1732,7 +1789,11 @@ class PostgresNode(object):
         if options:
             _params.extend(options)
 
-        execute_utility2(self._os_ops, _params, self.utils_log_file)
+        utils.execute_utility3(
+            self._os_ops,
+            _params,
+            self.utils_log_file,
+        )
 
         return filename
 
@@ -1761,7 +1822,11 @@ class PostgresNode(object):
 
         # try pg_restore if dump is binary format, and psql if not
         try:
-            execute_utility2(self._os_ops, _params, self.utils_log_file)
+            utils.execute_utility3(
+                self._os_ops,
+                _params,
+                self.utils_log_file,
+            )
         except ExecUtilException:
             self.psql(filename=filename, dbname=dbname, username=username)
 
@@ -2002,12 +2067,14 @@ class PostgresNode(object):
                             dbname=dbname, username=username, **params)
         # yapf: enable
 
-    def pgbench(self,
-                dbname=None,
-                username=None,
-                stdout=None,
-                stderr=None,
-                options=None):
+    def pgbench(
+        self,
+        dbname=None,
+        username=None,
+        stdout=None,
+        stderr=None,
+        options=None,
+    ) -> OsProcessController:
         """
         Spawn a pgbench process.
 
@@ -2019,7 +2086,7 @@ class PostgresNode(object):
             options: additional options for pgbench (list).
 
         Returns:
-            Process created by subprocess.Popen.
+            OsProcessController.
         """
         if options is None:
             options = []
@@ -2036,10 +2103,14 @@ class PostgresNode(object):
         # should be the last one
         _params.append(dbname)
 
-        proc = self._os_ops.exec_command(_params, stdout=stdout, stderr=stderr, get_process=True)
+        proc = self._os_ops.popen(
+            _params,
+            stdout=stdout,
+            stderr=stderr,
+        )
 
         # [2026-06-21] It is so
-        assert isinstance(proc, subprocess.Popen)
+        assert isinstance(proc, OsProcessController)
         return proc
 
     def pgbench_with_wait(self,
@@ -2122,7 +2193,11 @@ class PostgresNode(object):
         # should be the last one
         _params.append(dbname)
 
-        return execute_utility2(self._os_ops, _params, self.utils_log_file)
+        return utils.execute_utility3(
+            self._os_ops,
+            _params,
+            self.utils_log_file,
+        ).stdout
 
     def connect(self,
                 dbname=None,
@@ -2295,7 +2370,22 @@ class PostgresNode(object):
         ]
         upgrade_command += options
 
-        return self._os_ops.exec_command(upgrade_command, expect_error=expect_error)
+        r: typing.Optional[typing.Any] = None
+        try:
+            r = self._os_ops.run(upgrade_command).stdout
+        except BaseException as e:
+            if not expect_error:
+                raise
+
+            logging.info("Exception ({}): {}".format(
+                type(e).__name__,
+                e,
+            ))
+
+        if expect_error:
+            raise RuntimeError("Operation executed without any errors.")
+
+        return r
 
     def _release_resources(self):
         self._free_port()