testgres 1.16.2__tar.gz → 1.16.3__tar.gz

{testgres-1.16.2/testgres.egg-info → testgres-1.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testgres
-Version: 1.16.2
+Version: 1.16.3
 Summary: Testing utility for PostgreSQL and its extensions
 Author-email: Postgres Professional <testgres@postgrespro.ru>
 License: PostgreSQL
@@ -24,7 +24,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pg8000
 Requires-Dist: port-for>=0.4
-Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
 Requires-Dist: testgres.os_ops<4.0.0,>=3.3.3

{testgres-1.16.2 → testgres-1.16.3}/pyproject.toml

@@ -62,7 +62,6 @@ classifiers = [
 dependencies = [
     "pg8000",
     "port-for>=0.4",
-    "six>=1.9.0",
     "psutil",
     "packaging",
     "testgres.os_ops>=3.3.3,<4.0.0",

{testgres-1.16.2 → testgres-1.16.3}/src/__init__.py

@@ -60,7 +60,7 @@ from testgres.operations.os_ops import OsProcessController
 from testgres.operations.local_ops import LocalOperations
 from testgres.operations.remote_ops import RemoteOperations
 
-__version__ = "1.16.2"
+__version__ = "1.16.3"
 
 __all__ = [
     "get_new_node",

{testgres-1.16.2 → testgres-1.16.3}/src/backup.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-from six import raise_from
-
 from .enums import XLogMethod
 
 from .consts import \
@@ -22,7 +20,7 @@ from .utils import \
     clean_on_error
 
 
-class NodeBackup(object):
+class NodeBackup:
     """
     Smart object responsible for backups
     """
@@ -131,7 +129,7 @@ class NodeBackup(object):
                 # Copy backup to new data dir
                 self.os_ops.copytree(data1, data2)
             except Exception as e:
-                raise_from(BackupException('Failed to copy files'), e)
+                raise BackupException('Failed to copy files') from e
         else:
             dest_base_dir = self.base_dir
 

{testgres-1.16.2 → testgres-1.16.3}/src/cache.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-from six import raise_from
-
 from .config import testgres_config
 
 from .consts import XLOG_CONTROL_FILE
@@ -49,7 +47,7 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = No
                 log,
             )
         except ExecUtilException as e:
-            raise_from(InitNodeException("Failed to run initdb"), e)
+            raise InitNodeException("Failed to run initdb") from e
 
     if params or not testgres_config.cache_initdb or not cached:
         call_initdb(data_dir, logfile)
@@ -89,7 +87,7 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = No
 
         except ExecUtilException as e:
             msg = "Failed to reset WAL for system id"
-            raise_from(InitNodeException(msg), e)
+            raise InitNodeException(msg) from e
 
         except Exception as e:
-            raise_from(InitNodeException("Failed to spawn a node"), e)
+            raise InitNodeException("Failed to spawn a node") from e

{testgres-1.16.2 → testgres-1.16.3}/src/config.py

@@ -17,7 +17,7 @@ log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(messag
 logging.basicConfig(level=log_level, format=log_format)
 
 
-class GlobalConfig(object):
+class GlobalConfig:
     """
     Global configuration object which allows user to override default settings.
     """

{testgres-1.16.2 → testgres-1.16.3}/src/connection.py

@@ -25,7 +25,7 @@ ProgrammingError = pglib.ProgrammingError
 OperationalError = pglib.OperationalError
 
 
-class NodeConnection(object):
+class NodeConnection:
     """
     Transaction wrapper returned by Node
     """

{testgres-1.16.2 → testgres-1.16.3}/src/consts.py

@@ -2,6 +2,7 @@
 
 # binary names
 BINARY_NAME__POSTGRES = "postgres"
+BINARY_NAME__PG_CTL = "pg_ctl"
 
 # names for dirs in base_dir
 DATA_DIR = "data"
@@ -44,5 +45,7 @@ MAX_WAL_SENDERS = 10
 LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60
 
 PG_CTL__STATUS__OK = 0
+
+PG_CTL__STATUS__FAILED = 1
 PG_CTL__STATUS__NODE_IS_STOPPED = 3
 PG_CTL__STATUS__BAD_DATADIR = 4

{testgres-1.16.2 → testgres-1.16.3}/src/decorators.py

@@ -1,4 +1,3 @@
-import six
 import functools
 
 
@@ -18,7 +17,7 @@ def positional_args_hack(*special_cases):
 
     for case in special_cases:
         k = len(case)
-        assert k not in six.iterkeys(cases), 'len must be unique'
+        assert k not in cases, 'len must be unique'
         cases[k] = case
 
     def decorator(function):
@@ -26,7 +25,7 @@ def positional_args_hack(*special_cases):
         def wrapper(*args, **kwargs):
             k = len(args)
 
-            if k in six.iterkeys(cases):
+            if k in cases:
                 case = cases[k]
 
                 for i in range(0, k):

{testgres-1.16.2 → testgres-1.16.3}/src/enums.py

@@ -1,5 +1,4 @@
 from enum import Enum, IntEnum
-from six import iteritems
 from psutil import NoSuchProcess
 
 
@@ -83,7 +82,7 @@ class ProcessType(Enum):
             if cmdline.startswith(ptype.value.replace(' ', '')):
                 return ptype
 
-        for ptype, names in iteritems(alternative_names):
+        for ptype, names in alternative_names.items():
             for name in names:
                 if cmdline.startswith(name.replace(' ', '')):
                     return ptype

{testgres-1.16.2 → testgres-1.16.3}/src/exceptions.py

@@ -1,6 +1,5 @@
 # coding: utf-8
 
-import six
 import typing
 
 from testgres.operations.exceptions import TestgresException
@@ -44,7 +43,6 @@ class PortForException(TestgresException):
         return result
 
 
-@six.python_2_unicode_compatible
 class QueryException(TestgresException):
     _description: typing.Optional[str]
     _query: typing.Optional[str]
@@ -76,7 +74,7 @@ class QueryException(TestgresException):
         if self._query:
             msg.append(u'Query: {}'.format(self._query))
 
-        r = six.text_type('\n').join(msg)
+        r = '\n'.join(msg)
         assert type(r) is str
         return r
 
@@ -162,7 +160,6 @@ class CatchUpException(TestgresException):
         return result
 
 
-@six.python_2_unicode_compatible
 class StartNodeException(TestgresException):
     _description: typing.Optional[str]
     _files: typing.Optional[typing.Iterable]
@@ -196,7 +193,7 @@ class StartNodeException(TestgresException):
             assert type(lines) in [str, bytes]
             msg.append(u'{}\n----\n{}\n'.format(f, lines))
 
-        return six.text_type('\n').join(msg)
+        return '\n'.join(msg)
 
     @property
     def description(self) -> typing.Optional[str]:

testgres-1.16.3/src/impl/internal_node_utils.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from . import internal_utils
+from . platforms.internal_platform_utils_provider import InternalPlaformUtilsProvider
+
+from .. import consts
+from ..node_state import PostgresNodeState
+from ..enums import NodeStatus
+from ..raise_error import RaiseError
+from ..exceptions import ExecUtilException
+
+from testgres.operations.os_ops import OsOperations
+
+import typing
+import time
+
+
+class InternalNodeUtils:
+    T_PLATFORM_UTILS = InternalPlaformUtilsProvider.T_PLATFORM_UTILS
+
+    @staticmethod
+    def get_pg_node_state(
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str,
+        utils_log_file: typing.Optional[str],
+    ) -> PostgresNodeState:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert utils_log_file is None or type(utils_log_file) is str
+
+        C_MAX_ATTEMPTS = 3
+        C_SLEEP_TIME1 = 1
+        C_SLEEP_TIME_MULT = 2
+
+        pg_ctl_params = [
+            os_ops.build_path(bin_dir, consts.BINARY_NAME__PG_CTL),
+            "-D",
+            data_dir,
+            "status",
+        ]
+
+        attempt = 0
+        sleep_time = C_SLEEP_TIME1
+
+        platform_utils_provider = InternalPlaformUtilsProvider(
+            os_ops,
+        )
+
+        while True:
+            assert type(attempt) is int
+            assert attempt >= 0
+            assert attempt < C_MAX_ATTEMPTS
+
+            attempt += 1
+
+            if attempt > 1:
+                internal_utils.send_log_debug("Sleep {} second(s) before an attempt #{}".format(
+                    sleep_time,
+                    attempt,
+                ))
+                time.sleep(sleep_time)
+                sleep_time = sleep_time * C_SLEEP_TIME_MULT
+
+            exec_r = internal_utils.execute_utility3(
+                os_ops,
+                pg_ctl_params,
+                utils_log_file,
+                check=False,
+            )
+
+            status_code = exec_r.returncode
+            out = exec_r.stdout
+            error = exec_r.stderr
+
+            assert type(status_code) is int
+            assert type(out) is str
+            assert type(error) is str
+
+            # -----------------
+            if status_code == consts.PG_CTL__STATUS__NODE_IS_STOPPED:
+                return PostgresNodeState(NodeStatus.Stopped, None)
+
+            # -----------------
+            if status_code == consts.PG_CTL__STATUS__BAD_DATADIR:
+                return PostgresNodeState(NodeStatus.Uninitialized, None)
+
+            # -----------------
+            if status_code == consts.PG_CTL__STATUS__OK:
+                pid = __class__._parse_pid(
+                    out,
+                    pg_ctl_params,
+                )
+                assert type(pid) is int
+                assert pid != 0
+
+                # ----------------- detect zombie
+                if platform_utils_provider.get().ProcessIsZombie_soft_check(os_ops, pid) is True:
+                    internal_utils.send_log_debug("Postmaster process {} is a zombie.".format(
+                        pid,
+                    ))
+                    return PostgresNodeState(NodeStatus.Zombie, pid)
+
+                # -----------------
+                return PostgresNodeState(NodeStatus.Running, pid)
+
+            assert status_code != consts.PG_CTL__STATUS__OK
+
+            errMsg = "Getting of a node status [data_dir is {0}] failed.".format(
+                data_dir,
+            )
+
+            e1 = ExecUtilException(
+                message=errMsg,
+                command=pg_ctl_params,
+                exit_code=status_code,
+                out=out,
+                error=error,
+            )
+
+            if status_code == consts.PG_CTL__STATUS__FAILED:
+                internal_utils.send_log_debug(
+                    "pg_ctl fails with an error: {}".format(
+                        exec_r.stderr,
+                    ))
+
+                try:
+                    find_postmaster_r = platform_utils_provider.get().FindPostmaster(
+                        os_ops,
+                        bin_dir,
+                        data_dir,
+                    )
+                except Exception as e2:
+                    raise e2 from e1
+
+                assert type(find_postmaster_r) is __class__.T_PLATFORM_UTILS.FindPostmasterResult
+
+                if find_postmaster_r.code == __class__.T_PLATFORM_UTILS.FindPostmasterResultCode.ok:
+                    # Postmaster is alive. Let's wait a few seconds and check its status again.
+                    internal_utils.send_log_debug(
+                        "Postmaster is found and has PID {}.".format(
+                            find_postmaster_r.pid,
+                        ))
+
+                    if attempt < C_MAX_ATTEMPTS:
+                        continue
+
+            raise e1
+
+    @staticmethod
+    def _parse_pid(
+        out: str,
+        pg_ctl_params,
+    ) -> int:
+        assert type(out) is str
+
+        if out == "":
+            RaiseError.pg_ctl_returns_an_empty_string(
+                pg_ctl_params,
+            )
+
+        C_PID_PREFIX = "(PID: "
+
+        i = out.find(C_PID_PREFIX)
+
+        if i == -1:
+            RaiseError.pg_ctl_returns_an_unexpected_string(
+                out,
+                pg_ctl_params,
+            )
+
+        assert i > 0
+        assert i < len(out)
+        assert len(C_PID_PREFIX) <= len(out)
+        assert i <= len(out) - len(C_PID_PREFIX)
+
+        i += len(C_PID_PREFIX)
+        start_pid_s = i
+
+        while True:
+            if i == len(out):
+                RaiseError.pg_ctl_returns_an_unexpected_string(
+                    out,
+                    pg_ctl_params,
+                )
+
+            ch = out[i]
+
+            if ch == ")":
+                break
+
+            if ch.isdigit():
+                i += 1
+                continue
+
+            RaiseError.pg_ctl_returns_an_unexpected_string(
+                out,
+                pg_ctl_params,
+            )
+            assert False
+
+        if i == start_pid_s:
+            RaiseError.pg_ctl_returns_an_unexpected_string(
+                out,
+                pg_ctl_params,
+            )
+
+        # TODO: Let's verify a length of pid string.
+
+        pid = int(out[start_pid_s:i])
+
+        if pid == 0:
+            RaiseError.pg_ctl_returns_a_zero_pid(
+                out,
+                pg_ctl_params,
+            )
+
+        assert pid != 0
+
+        return pid

{testgres-1.16.2 → testgres-1.16.3}/src/impl/internal_utils.py

@@ -1,4 +1,10 @@
 from testgres.operations.os_ops import OsOperations
+from testgres.operations.os_ops import OsCommandResult
+from testgres.operations.types import T_OS_CMD
+from testgres.operations.types import T_OS_EXEC_ENV
+from testgres.operations.helpers import Helpers as OsHelpers
+
+from ..exceptions import ExecUtilException
 
 import logging
 import typing
@@ -90,3 +96,55 @@ def read_line_to_pos__bin(
     assert type(result) is bytes
     assert len(result) <= (position - read_position)
     return result
+
+
+def execute_utility3(
+    os_ops: OsOperations,
+    args: T_OS_CMD,
+    logfile: typing.Optional[str] = None,
+    check: bool = True,
+    exec_env: typing.Optional[T_OS_EXEC_ENV] = None,
+) -> OsCommandResult:
+    assert os_ops is not None
+    assert isinstance(os_ops, OsOperations)
+    assert type(check) is bool
+    assert exec_env is None or type(exec_env) is dict
+
+    exec_r = os_ops.run(
+        args,
+        check=check,
+        encoding=OsHelpers.GetDefaultEncoding(),
+        exec_env=exec_env,
+    )
+
+    assert type(exec_r) is OsCommandResult
+
+    # write new log entry if possible
+    if logfile:
+        try:
+            log_lines = [
+                os_ops.join_command_arguments(args),
+            ]
+
+            if exec_r.stdout is None:
+                log_lines.append("# #NONE#")
+            else:
+                # comment-out lines
+                assert type(exec_r.stdout) is str
+                log_lines += ['# ' + line for line in exec_r.stdout.splitlines()]
+
+            log_lines.append("")
+
+            os_ops.write(
+                filename=logfile,
+                data="\n".join(log_lines),
+                truncate=False,
+            )
+        except IOError:
+            raise ExecUtilException(
+                "Problem with writing to logfile `{}` during run command `{}`".format(
+                    logfile,
+                    args,
+                ))
+
+    return exec_r

{testgres-1.16.2 → testgres-1.16.3}/src/impl/platforms/darwin/internal_platform_utils.py

@@ -248,7 +248,7 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
         )
 
     # --------------------------------------------------------------------
-    def ProcessIsZombi_soft_check(
+    def ProcessIsZombie_soft_check(
         self,
         os_ops: OsOperations,
         pid: int,

{testgres-1.16.2 → testgres-1.16.3}/src/impl/platforms/internal_platform_utils.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from ...raise_error import RaiseError
+
 import enum
 import typing
 
@@ -59,13 +61,13 @@ class InternalPlatformUtils:
         assert isinstance(os_ops, OsOperations)
         assert type(bin_dir) is str
         assert type(data_dir) is str
-        raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.")
+        RaiseError.method_is_not_implemented(__class__, "FindPostmaster")
 
-    def ProcessIsZombi_soft_check(
+    def ProcessIsZombie_soft_check(
         self,
         os_ops: OsOperations,
         pid: int,
     ) -> typing.Optional[bool]:
         assert isinstance(os_ops, OsOperations)
         assert type(pid) is int
-        raise NotImplementedError("InternalPlatformUtils::ProcessIsZombi_soft_ver is not implemented.")
+        RaiseError.method_is_not_implemented(__class__, "ProcessIsZombie_soft_check")

testgres-1.16.3/src/impl/platforms/internal_platform_utils_provider.py

@@ -0,0 +1,32 @@
+from .internal_platform_utils_factory import create_internal_platform_utils
+from .internal_platform_utils_factory import InternalPlatformUtils
+
+from testgres.operations.os_ops import OsOperations
+
+import typing
+
+
+class InternalPlaformUtilsProvider:
+    T_PLATFORM_UTILS = InternalPlatformUtils
+
+    _os_ops: OsOperations
+    _platform_utils: typing.Optional[T_PLATFORM_UTILS] = None
+
+    def __init__(
+        self,
+        os_ops: OsOperations,
+    ):
+        assert isinstance(os_ops, OsOperations)
+        self._os_ops = os_ops
+        self._platform_utils = None
+        return
+
+    def get(self) -> T_PLATFORM_UTILS:
+        if self._platform_utils is None:
+            self._platform_utils = create_internal_platform_utils(
+                self._os_ops,
+            )
+            assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS)
+
+        assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS)
+        return self._platform_utils

{testgres-1.16.2 → testgres-1.16.3}/src/impl/platforms/linux/internal_platform_utils.py

@@ -247,7 +247,7 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
         )
 
     # --------------------------------------------------------------------
-    def ProcessIsZombi_soft_check(
+    def ProcessIsZombie_soft_check(
         self,
         os_ops: OsOperations,
         pid: int,
@@ -264,7 +264,10 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
 
         try:
             # Read one line from /proc/PID/stat
-            stat_content = os_ops.read_binary(proc_stat_file, 0).decode("utf-8", errors="ignore")
+            stat_content_b = os_ops.read_binary(proc_stat_file, 0)
+            assert type(stat_content_b) is bytes
+
+            stat_content = stat_content_b.decode("utf-8", errors="ignore")
 
             # We look for the closing parenthesis of the process name to ensure that
             # we start from it and not depend on spaces inside the parentheses!
@@ -281,19 +284,23 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
                 result = proc_status == "Z"
         except Exception as e:
             # If the file disappeared right during reading, it means the process is completely erased
-            if __class__._is_file_not_found_exception(e):
+            if __class__._is_zombie_file_exception(e):
                 result = False
 
         return result
 
     @staticmethod
-    def _is_file_not_found_exception(e: Exception) -> bool:
+    def _is_zombie_file_exception(e: Exception) -> bool:
         if isinstance(e, FileNotFoundError):
             return True
 
+        if isinstance(e, ProcessLookupError):
+            return True
+
         if isinstance(e, ExecUtilException):
-            if e.exit_code == 2:
+            if e.exit_code == 1:
                 return True
+            return False
 
         return False
 

{testgres-1.16.2 → testgres-1.16.3}/src/impl/platforms/win32/internal_platform_utils.py

@@ -17,7 +17,7 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
         assert type(data_dir) is str
         return __class__.FindPostmasterResult.create_not_implemented()
 
-    def ProcessIsZombi_soft_check(
+    def ProcessIsZombie_soft_check(
         self,
         os_ops: OsOperations,
         pid: int,