testgres 1.14.4__tar.gz → 1.15.0__tar.gz

{testgres-1.14.4/testgres.egg-info → testgres-1.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testgres
-Version: 1.14.4
+Version: 1.15.0
 Summary: Testing utility for PostgreSQL and its extensions
 Author-email: Postgres Professional <testgres@postgrespro.ru>
 License: PostgreSQL
@@ -27,7 +27,7 @@ Requires-Dist: port-for>=0.4
 Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
-Requires-Dist: testgres.os_ops<3.0.0,>=2.4.0
+Requires-Dist: testgres.os_ops<4.0.0,>=3.1.0
 Dynamic: license-file
 
 [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/ci.yml)

{testgres-1.14.4 → testgres-1.15.0}/pyproject.toml

@@ -65,7 +65,7 @@ dependencies = [
     "six>=1.9.0",
     "psutil",
     "packaging",
-    "testgres.os_ops>=2.4.0,<3.0.0",
+    "testgres.os_ops>=3.1.0,<4.0.0",
 ]
 
 [project.urls]

{testgres-1.14.4 → testgres-1.15.0}/src/__init__.py

@@ -41,8 +41,8 @@ from .node_app import NodeApp
 from .utils import \
     reserve_port, \
     release_port, \
-    bound_ports, \
     get_bin_path, \
+    get_bin_dir, \
     get_pg_config, \
     get_pg_version, \
     parse_pg_version
@@ -57,7 +57,7 @@ from testgres.operations.os_ops import OsOperations, ConnectionParams
 from testgres.operations.local_ops import LocalOperations
 from testgres.operations.remote_ops import RemoteOperations
 
-__version__ = "1.14.4"
+__version__ = "1.15.0"
 
 __all__ = [
     "get_new_node",
@@ -72,7 +72,7 @@ __all__ = [
     "NodeApp",
     "PostgresNode",
     "PortManager",
-    "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "parse_pg_version",
+    "reserve_port", "release_port", "get_bin_path", "get_bin_dir", "get_pg_config", "get_pg_version", "parse_pg_version",
     "First", "Any",
     "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"
 ]

{testgres-1.14.4 → testgres-1.15.0}/src/connection.py

@@ -14,7 +14,7 @@ from .enums import IsolationLevel
 
 from .defaults import \
     default_dbname, \
-    default_username
+    default_username2
 
 from .exceptions import QueryException
 
@@ -38,7 +38,7 @@ class NodeConnection(object):
 
         # Set default arguments
         dbname = dbname or default_dbname()
-        username = username or default_username()
+        username = username or default_username2(node.os_ops)
 
         self._node = node
 

{testgres-1.14.4 → testgres-1.15.0}/src/consts.py

@@ -10,6 +10,10 @@ TMP_DUMP = 'tgsd_'
 TMP_CACHE = 'tgsc_'
 TMP_BACKUP = 'tgsb_'
 
+TMP_TESTGRES = "testgres"
+
+TMP_TESTGRES_PORTS = TMP_TESTGRES + "/ports"
+
 # path to control file
 XLOG_CONTROL_FILE = "global/pg_control"
 

{testgres-1.14.4 → testgres-1.15.0}/src/defaults.py

@@ -1,6 +1,9 @@
 import datetime
 import struct
 import uuid
+import typing
+
+from testgres.operations.os_ops import OsOperations
 
 from .config import testgres_config as tconf
 
@@ -13,11 +16,30 @@ def default_dbname():
     return 'postgres'
 
 
-def default_username():
+def default_username(os_ops: typing.Optional[OsOperations] = None) -> str:
     """
     Return default username (current user).
     """
-    return tconf.os_ops.get_user()
+    assert os_ops is None or isinstance(os_ops, OsOperations)
+
+    if os_ops is None:
+        os_ops = tconf.os_ops
+
+    assert isinstance(os_ops, OsOperations)
+    result = default_username2(os_ops)
+    assert type(result) is str
+    return result
+
+
+def default_username2(os_ops: OsOperations) -> str:
+    """
+    Return default username (current user).
+    """
+    assert isinstance(os_ops, OsOperations)
+
+    result = os_ops.get_user()
+    assert type(result) is str
+    return result
 
 
 def generate_app_name():

{testgres-1.14.4 → testgres-1.15.0}/src/enums.py

@@ -29,7 +29,7 @@ class NodeStatus(IntEnum):
     Status of a PostgresNode
     """
 
-    Running, Stopped, Uninitialized = range(3)
+    Running, Stopped, Uninitialized, Zombie = range(4)
 
     # for Python 3.x
     def __bool__(self):

testgres-1.15.0/src/impl/file_line_reader.py

@@ -0,0 +1,80 @@
+# coding: utf-8
+
+import typing
+
+from testgres.operations.os_ops import OsOperations
+
+
+class FileLineReader:
+    _os_ops: OsOperations
+    _file_name: str
+    _file_encoding: str
+    _file_pos: int
+    _buffer_pos: int
+    _buffer: bytes
+
+    # --------------------------------------------------------------------
+    def __init__(
+        self,
+        os_ops: OsOperations,
+        file_name: str,
+        file_encoding: str = "utf-8",
+    ):
+        assert isinstance(os_ops, OsOperations)
+        assert type(file_encoding) is str
+        self._os_ops = os_ops
+        self._file_name = file_name
+        self._file_encoding = file_encoding
+        self._file_pos = 0
+        self._buffer_pos = 0
+        self._buffer = b''
+        return
+
+    # interface ----------------------------------------------------------
+    def read_line(self) -> typing.Optional[str]:
+        assert isinstance(self._os_ops, OsOperations)
+        assert type(self._buffer_pos) is int
+        assert type(self._buffer) is bytes
+        assert self._buffer_pos >= 0
+        assert self._buffer_pos <= len(self._buffer)
+
+        scan_pos = self._buffer_pos
+
+        while True:
+            sz1 = len(self._buffer)
+
+            if scan_pos == sz1:
+                block = self._os_ops.read_binary(
+                    self._file_name,
+                    self._file_pos,
+                )
+                assert type(block) is bytes
+                self._buffer += block
+                self._file_pos += len(block)
+
+            x = self._buffer.find(b'\n', scan_pos)
+
+            sz2 = len(self._buffer)
+
+            if x == -1:
+                if scan_pos == sz2:
+                    return None
+
+                if self._buffer_pos == 0:
+                    scan_pos = sz2
+                else:
+                    assert self._buffer_pos > 0
+                    self._buffer = self._buffer[self._buffer_pos:]
+                    scan_pos = sz2 - self._buffer_pos
+                    self._buffer_pos = 0
+                continue
+
+            assert x >= 0
+            assert x < sz2
+
+            b = self._buffer[self._buffer_pos:(x+1)]
+
+            s = b.decode(self._file_encoding)
+
+            self._buffer_pos = x + 1
+            return s

{testgres-1.14.4 → testgres-1.15.0}/src/impl/platforms/internal_platform_utils.py

@@ -60,3 +60,12 @@ class InternalPlatformUtils:
         assert type(bin_dir) is str
         assert type(data_dir) is str
         raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.")
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+        raise NotImplementedError("InternalPlatformUtils::ProcessIsZombi_soft_ver is not implemented.")

testgres-1.15.0/src/impl/platforms/linux/internal_platform_utils.py

@@ -0,0 +1,429 @@
+from __future__ import annotations
+
+from .. import internal_platform_utils as base
+from ... import internal_utils
+from ....raise_error import RaiseError
+
+from testgres.operations.os_ops import OsOperations
+from testgres.operations.exceptions import ExecUtilException
+
+import re
+import shlex
+import typing
+import time
+
+
+class InternalPlatformUtils(base.InternalPlatformUtils):
+    C_MAX_FIND_POSTMASTER_ATTEMPTS = 5
+    C_BASH_EXE = "/bin/bash"
+
+    sm_exec_env = {
+        "LANG": "en_US.UTF-8",
+        "LC_ALL": "en_US.UTF-8",
+    }
+
+    # --------------------------------------------------------------------
+    def FindPostmaster(
+        self,
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> InternalPlatformUtils.FindPostmasterResult:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert type(__class__.C_BASH_EXE) is str
+        assert type(__class__.sm_exec_env) is dict
+        assert len(__class__.C_BASH_EXE) > 0
+        assert len(bin_dir) > 0
+        assert len(data_dir) > 0
+
+        failures: typing.List[Exception] = []
+
+        postmaster_pid: typing.Optional[int] = None
+
+        nAttempts = 0
+
+        while True:
+            nAttempts += 1
+
+            try:
+                postmaster_pid = __class__._FindPostmaster(
+                    os_ops,
+                    bin_dir,
+                    data_dir,
+                )
+            except Exception as e:
+                failures.append(e)
+
+                log_msg = "FindPostmaster (bin_dir={!r}, data_dir={!r}) detects a problem. Exception {}:\n{}".format(
+                    bin_dir,
+                    data_dir,
+                    type(e).__name__,
+                    e,
+                )
+                internal_utils.send_log_debug(log_msg)
+
+                if nAttempts < __class__.C_MAX_FIND_POSTMASTER_ATTEMPTS:
+                    time.sleep(0.05)
+                    continue
+
+                __class__._find_postmaster__throw_error__fail(
+                    bin_dir=bin_dir,
+                    data_dir=data_dir,
+                    failures=failures,
+                )
+
+            break
+
+        if postmaster_pid is None:
+            return InternalPlatformUtils.FindPostmasterResult.create_not_found()
+
+        assert type(postmaster_pid) is int
+
+        return InternalPlatformUtils.FindPostmasterResult.create_ok(postmaster_pid)
+
+    # --------------------------------------------------------------------
+    @staticmethod
+    def _FindPostmaster(
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> typing.Optional[int]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert type(__class__.C_BASH_EXE) is str
+        assert type(__class__.sm_exec_env) is dict
+        assert len(__class__.C_BASH_EXE) > 0
+        assert len(bin_dir) > 0
+        assert len(data_dir) > 0
+
+        pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres"))
+        data_dir_e = re.escape(data_dir)
+
+        assert type(pg_path_e) is str
+        assert type(data_dir_e) is str
+
+        regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?"
+
+        cmd = [
+            __class__.C_BASH_EXE,
+            "-c",
+            "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp),
+           ]
+
+        exec_r = os_ops.exec_command(
+            cmd=cmd,
+            ignore_errors=True,
+            verbose=True,
+            exec_env=__class__.sm_exec_env,
+        )
+
+        assert type(exec_r) is tuple
+        assert len(exec_r) == 3
+
+        exit_status, output_b, error_b = exec_r
+
+        assert type(exit_status) is int
+        assert type(output_b) is bytes
+        assert type(error_b) is bytes
+
+        if exit_status == 1:
+            return None
+
+        output = output_b.decode("utf-8")
+        error = error_b.decode("utf-8")
+
+        assert type(output) is str
+        assert type(error) is str
+
+        if exit_status != 0:
+            errMsg = f"test command returned an unexpected exit code: {exit_status}"
+            raise ExecUtilException(
+                message=errMsg,
+                command=cmd,
+                exit_code=exit_status,
+                out=output,
+                error=error,
+            )
+
+        lines = output.splitlines()
+        assert type(lines) is list
+
+        if len(lines) == 0:
+            # ACHTUNG!
+            raise RuntimeError("Command returns 0 error code without output.")
+
+        # parse result lines
+        pid_to_ppid: __class__.T_PID_TO_PPID = {}
+
+        for i_line in range(len(lines)):
+            assert type(lines[i_line]) is str
+
+            parts = lines[i_line].split()
+            assert type(parts) is list
+
+            if len(parts) < 2:
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "no usefull data",
+                )
+
+            if not parts[0].isdigit():
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "bad pid",
+                )
+
+            if not parts[1].isdigit():
+                __class__._find_postmaster__throw_error__bad_line_format(
+                    lines,
+                    i_line,
+                    "bad ppid",
+                )
+
+            pid = int(parts[0])
+            ppid = int(parts[1])
+
+            if pid not in pid_to_ppid:
+                pid_to_ppid[pid] = ppid
+                continue
+
+            other_ppid = pid_to_ppid[pid]
+            assert type(other_ppid) is int
+
+            if ppid == other_ppid:
+                log_msg = "FindPostmaster (data_dir={!r}) get pid ({}) with ppid ({}) more than one time.".format(
+                    data_dir,
+                    pid,
+                    ppid,
+                )
+                internal_utils.send_log_debug(log_msg)
+                continue
+
+            # ACTUNG ppid is changed --> restart
+            __class__._find_postmaster__throw_error__ppid_is_changed(
+                pid,
+                ppid,
+                other_ppid,
+                lines,
+            )
+
+        assert len(pid_to_ppid) <= len(lines)
+
+        true_postmasters = [
+            pid for pid, ppid in pid_to_ppid.items()
+            if ppid not in pid_to_ppid
+        ]
+
+        if len(true_postmasters) == 0:
+            __class__._find_postmaster__throw_error__cycle(
+                pid_to_ppid,
+            )
+
+        if len(true_postmasters) == 1:
+            true_pid = true_postmasters[0]
+
+            if len(pid_to_ppid) > 1:
+                msg = "Many processes like a postmaster for data dir [{}] are found ({}).".format(
+                    data_dir,
+                    len(true_postmasters),
+                )
+
+                msg += " List (ppid->pid): {}.".format(
+                    __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+                )
+
+                msg += " True postmaster PID is {}.".format(true_pid)
+                internal_utils.send_log_debug(msg)
+
+            return true_pid
+
+        assert len(true_postmasters) > 1
+
+        __class__._find_postmaster__throw_error__many_postmasters(
+            true_postmasters,
+            pid_to_ppid,
+        )
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+
+        proc_stat_file = os_ops.build_path("/proc", str(pid), "stat")
+
+        if not os_ops.path_exists(proc_stat_file):
+            return False
+
+        result: typing.Optional[bool] = None
+
+        try:
+            # Read one line from /proc/PID/stat
+            stat_content = os_ops.read_binary(proc_stat_file, 0).decode("utf-8", errors="ignore")
+
+            # We look for the closing parenthesis of the process name to ensure that
+            # we start from it and not depend on spaces inside the parentheses!
+            r_paren_idx = stat_content.rfind(")")
+
+            if r_paren_idx == -1:
+                pass
+            elif len(stat_content) <= r_paren_idx + 2:
+                pass
+            else:
+                # The status goes exactly one space after the closing bracket
+                assert (r_paren_idx + 2) < len(stat_content)
+                proc_status = stat_content[r_paren_idx + 2]
+                result = proc_status == "Z"
+        except Exception as e:
+            # If the file disappeared right during reading, it means the process is completely erased
+            if __class__._is_file_not_found_exception(e):
+                result = False
+
+        return result
+
+    @staticmethod
+    def _is_file_not_found_exception(e: Exception) -> bool:
+        if isinstance(e, FileNotFoundError):
+            return True
+
+        if isinstance(e, ExecUtilException):
+            if e.exit_code == 2:
+                return True
+
+        return False
+
+    T_PID_TO_PPID = typing.Dict[int, int]
+
+    @staticmethod
+    def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str:
+        result = ""
+        sep = ""
+        for pid, ppid in pid_to_ppid.items():
+            result += sep + " {}->{}".format(ppid, pid)
+            sep = ", "
+            continue
+        return result
+
+    @staticmethod
+    def _find_postmaster__throw_error__bad_line_format(
+        lines: typing.List[str],
+        i_line: int,
+        hint: str,
+    ) -> typing.NoReturn:
+        assert type(lines) is list
+        assert type(i_line) is int
+        assert type(hint) is int
+
+        error_lines: typing.List[str] = []
+        error_lines.append(
+            "Line {} has bad format. Hint: {}.".format(
+                i_line + 1,
+                hint,
+            ),
+        )
+        error_lines.append(
+            "Problem line is:"
+        )
+        error_lines.append(
+            " " + repr(lines[i_line]),
+        )
+        error_lines.append(
+            "All the lines is:",
+        )
+        for i in range(len(lines)):
+            error_lines.append(
+                " {}. {!r}".format(i + 1, lines[i]),
+            )
+            continue
+
+        raise RuntimeError("\n".join(error_lines))
+
+    @staticmethod
+    def _find_postmaster__throw_error__ppid_is_changed(
+        pid: int,
+        ppid: int,
+        other_ppid: int,
+        lines: typing.List[str],
+    ) -> typing.NoReturn:
+        assert type(pid) is int
+        assert type(ppid) is int
+        assert type(other_ppid) is int
+        assert type(lines) is list
+
+        error_lines: typing.List[str] = []
+        error_lines.append(
+            "Parent of process ({}) is changed from {} to {}.".format(
+                pid,
+                other_ppid,
+                ppid,
+            ),
+        )
+        error_lines.append(
+            "All the lines is:",
+        )
+        for i in range(len(lines)):
+            error_lines.append(
+                " {}. {!r}".format(i + 1, lines[i]),
+            )
+            continue
+
+        raise RuntimeError("\n".join(error_lines))
+
+    @staticmethod
+    def _find_postmaster__throw_error__cycle(
+        pid_to_ppid: T_PID_TO_PPID,
+    ) -> typing.NoReturn:
+        msg = "Cycle in processes postgres process tree. "
+
+        msg += " List (ppid->pid): {},".format(
+            __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+        )
+
+        msg += " List size is {}.".format(
+            len(pid_to_ppid),
+        )
+        raise RuntimeError(msg)
+
+    @staticmethod
+    def _find_postmaster__throw_error__many_postmasters(
+        postmaster_pids: typing.List[int],
+        pid_to_ppid: T_PID_TO_PPID,
+    ) -> typing.NoReturn:
+        msg = "Many processes like a postmaster are found ({}): {}.".format(
+            len(postmaster_pids),
+            ", ".join(map(str, postmaster_pids)),
+        )
+
+        msg += " Trees (ppid->pid): {}.".format(
+            __class__._make_text_from_pid_to_ppid(pid_to_ppid),
+        )
+
+        msg += " Total process count is {}.".format(len(pid_to_ppid))
+        raise RuntimeError(msg)
+
+    @staticmethod
+    def _find_postmaster__throw_error__fail(
+        bin_dir: str,
+        data_dir: str,
+        failures: typing.List[Exception],
+    ) -> typing.NoReturn:
+        assert type(bin_dir) is str
+        assert type(data_dir) is str
+        assert type(failures) is list
+
+        method_name = "InternalPlatformUtils::FindPostmaster(bin_dir={!r}, data_dir={!r})".format(
+            bin_dir,
+            data_dir,
+        )
+
+        RaiseError.function_did_multiple_attempts_without_stable_result(
+            method_name,
+            failures,
+        )

{testgres-1.14.4 → testgres-1.15.0}/src/impl/platforms/win32/internal_platform_utils.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from .. import internal_platform_utils as base
 from testgres.operations.os_ops import OsOperations
+import typing
 
 
 class InternalPlatformUtils(base.InternalPlatformUtils):
@@ -15,3 +16,12 @@ class InternalPlatformUtils(base.InternalPlatformUtils):
         assert type(bin_dir) is str
         assert type(data_dir) is str
         return __class__.FindPostmasterResult.create_not_implemented()
+
+    def ProcessIsZombi_soft_check(
+        self,
+        os_ops: OsOperations,
+        pid: int,
+    ) -> typing.Optional[bool]:
+        assert isinstance(os_ops, OsOperations)
+        assert type(pid) is int
+        return None