testgres 1.15.0__tar.gz → 1.15.2__tar.gz

{testgres-1.15.0/testgres.egg-info → testgres-1.15.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testgres
-Version: 1.15.0
+Version: 1.15.2
 Summary: Testing utility for PostgreSQL and its extensions
 Author-email: Postgres Professional <testgres@postgrespro.ru>
 License: PostgreSQL
@@ -27,7 +27,7 @@ Requires-Dist: port-for>=0.4
 Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
-Requires-Dist: testgres.os_ops<4.0.0,>=3.1.0
+Requires-Dist: testgres.os_ops<4.0.0,>=3.2.0
 Dynamic: license-file
 
 [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/ci.yml)

{testgres-1.15.0 → testgres-1.15.2}/pyproject.toml

@@ -65,7 +65,7 @@ dependencies = [
     "six>=1.9.0",
     "psutil",
     "packaging",
-    "testgres.os_ops>=3.1.0,<4.0.0",
+    "testgres.os_ops>=3.2.0,<4.0.0",
 ]
 
 [project.urls]

{testgres-1.15.0 → testgres-1.15.2}/src/__init__.py

@@ -57,7 +57,7 @@ from testgres.operations.os_ops import OsOperations, ConnectionParams
 from testgres.operations.local_ops import LocalOperations
 from testgres.operations.remote_ops import RemoteOperations
 
-__version__ = "1.15.0"
+__version__ = "1.15.2"
 
 __all__ = [
     "get_new_node",

{testgres-1.15.0 → testgres-1.15.2}/src/impl/file_line_reader.py

@@ -1,5 +1,7 @@
 # coding: utf-8
 
+from . import internal_utils
+
 import typing
 
 from testgres.operations.os_ops import OsOperations
@@ -19,15 +21,20 @@ class FileLineReader:
         os_ops: OsOperations,
         file_name: str,
         file_encoding: str = "utf-8",
+        file_pos: int = 0
     ):
         assert isinstance(os_ops, OsOperations)
         assert type(file_encoding) is str
         self._os_ops = os_ops
         self._file_name = file_name
         self._file_encoding = file_encoding
-        self._file_pos = 0
+        self._file_pos = file_pos
         self._buffer_pos = 0
-        self._buffer = b''
+        self._buffer = internal_utils.read_line_to_pos__bin(
+            os_ops,
+            file_name,
+            file_pos,
+        )
         return
 
     # interface ----------------------------------------------------------

testgres-1.15.2/src/impl/internal_utils.py

@@ -0,0 +1,92 @@
+from testgres.operations.os_ops import OsOperations
+
+import logging
+import typing
+
+
+def send_log(level: int, msg: str) -> None:
+    assert type(level) is int
+    assert type(msg) is str
+
+    return logging.log(level, "[testgres] " + msg)
+
+
+def send_log_info(msg: str) -> None:
+    assert type(msg) is str
+
+    return send_log(logging.INFO, msg)
+
+
+def send_log_debug(msg: str) -> None:
+    assert type(msg) is str
+
+    return send_log(logging.DEBUG, msg)
+
+
+def read_line_to_pos__bin(
+    os_ops: OsOperations,
+    filename: str,
+    position: int,
+) -> bytes:
+    assert type(filename) is str
+    assert type(position) is int
+    assert len(filename) > 0
+    assert position >= 0
+    assert os_ops is not None
+    assert isinstance(os_ops, OsOperations)
+
+    if position == 0:
+        return b''
+
+    assert position > 0
+
+    read_position = position
+    result_blocks: typing.List[bytes] = []
+
+    C_BACK_READ_BLOCK_SIZE = 4096
+
+    while read_position > 0:
+        if read_position < C_BACK_READ_BLOCK_SIZE:
+            block_sz = read_position
+        else:
+            block_sz = C_BACK_READ_BLOCK_SIZE
+
+        assert block_sz > 0
+        assert block_sz <= C_BACK_READ_BLOCK_SIZE
+
+        read_position -= block_sz
+
+        assert read_position < position
+        assert read_position >= 0
+
+        block = os_ops.read_binary(filename, read_position, block_sz)
+
+        assert type(block) is bytes
+
+        if len(block) != block_sz:
+            err_msg = "[BUG CHECK] Readed block has bad size ({}). Expected size is ({}). File name {}.".format(
+                len(block),
+                block_sz,
+                filename,
+            )
+            raise RuntimeError(err_msg)
+
+        assert len(block) == block_sz
+
+        x = block.rfind(b"\n", 0, block_sz)
+
+        if x == -1:
+            result_blocks.append(block)
+            continue
+
+        if x == block_sz - 1:
+            break
+
+        block = block[x + 1:]
+        result_blocks.append(block)
+        break
+
+    result = b''.join(reversed(result_blocks))
+    assert type(result) is bytes
+    assert len(result) <= (position - read_position)
+    return result

{testgres-1.15.0 → testgres-1.15.2}/src/logger.py

@@ -31,7 +31,7 @@ class TestgresLogger(threading.Thread):
         assert type(log_file_encoding) is str
         assert os_ops is None or isinstance(os_ops, OsOperations)
 
-        threading.Thread.__init__(self)
+        threading.Thread.__init__(self, daemon=True)
 
         if os_ops is None:
             os_ops = tconf.os_ops

{testgres-1.15.0 → testgres-1.15.2}/src/node.py

@@ -638,6 +638,12 @@ class PostgresNode(object):
         assert type(with_force) is bool
         assert max_attempts > 0
 
+        try:
+            self._try_shutdown_internal(max_attempts, with_force)
+        finally:
+            self._maybe_stop_logger()
+
+    def _try_shutdown_internal(self, max_attempts, with_force):
         attempts = 0
 
         # try stopping server N times
@@ -1278,7 +1284,7 @@ class PostgresNode(object):
                     if nAttempt == __class__._C_MAX_START_ATEMPTS:
                         self._raise_cannot_start_node(e, "Cannot start node after multiple attempts.")
 
-                    is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader)
+                    is_it_port_conflict = PostgresNodeUtils.detect_port_conflict(log_reader)
 
                     if not is_it_port_conflict:
                         LOCAL__raise_cannot_start_node__std(e)
@@ -1331,11 +1337,13 @@ class PostgresNode(object):
             "stop"
         ] + params  # yapf: disable
 
-        execute_utility2(self._os_ops, _params, self.utils_log_file)
-
-        self._manually_started_pm_pid = None
-
-        self._maybe_stop_logger()
+        try:
+            execute_utility2(self._os_ops, _params, self.utils_log_file)
+            self._manually_started_pm_pid = None
+        finally:
+            # always stop the reader thread, even if pg_ctl stop failed,
+            # so it can't leak into interpreter shutdown.
+            self._maybe_stop_logger()
         return self
 
     def kill(self, someone=None):
@@ -2430,9 +2438,14 @@ class PostgresNodeLogReader:
         position: int
         tail: bytes
 
-        def __init__(self, position: int):
+        def __init__(self, position: int, tail: bytes = b''):
+            assert type(position) is int
+            assert type(tail) is bytes
+            assert position >= 0
+
             self.position = position
-            self.tail = b''
+            self.tail = tail
+            return
 
     # --------------------------------------------------------------------
     class LogDataBlock:
@@ -2487,7 +2500,7 @@ class PostgresNodeLogReader:
         if from_beginnig:
             self._logs = dict()
         else:
-            self._logs = self._collect_logs()
+            self._logs = self._collect_logs(find_line_start=True)
 
         assert type(self._logs) is dict
         return
@@ -2496,13 +2509,13 @@ class PostgresNodeLogReader:
         assert self._node is not None
         assert isinstance(self._node, PostgresNode)
 
-        cur_logs: typing.Dict[str, __class__.LogInfo] = self._collect_logs()
+        cur_logs = self._collect_logs(find_line_start=False)
         assert cur_logs is not None
         assert type(cur_logs) is dict
 
         assert type(self._logs) is dict
 
-        result = list()
+        result: typing.List[__class__.LogDataBlock] = []
 
         for file_name, cur_log_info in cur_logs.items():
             assert type(file_name) is str
@@ -2570,7 +2583,8 @@ class PostgresNodeLogReader:
 
         return result
 
-    def _collect_logs(self) -> typing.Dict[str, LogInfo]:
+    def _collect_logs(self, find_line_start: bool) -> typing.Dict[str, LogInfo]:
+        assert type(find_line_start) is bool
         assert self._node is not None
         assert isinstance(self._node, PostgresNode)
 
@@ -2587,18 +2601,52 @@ class PostgresNodeLogReader:
             if not self._node.os_ops.path_exists(f):
                 continue
 
-            file_size = self._node.os_ops.get_file_size(f)
-            assert type(file_size) is int
-            assert file_size >= 0
-
-            result[f] = __class__.LogInfo(file_size)
+            result[f] = self._create_log_info(
+                self._node.os_ops,
+                f,
+                find_line_start,
+            )
+            continue
 
         return result
 
+    @staticmethod
+    def _create_log_info(
+        os_ops: OsOperations,
+        filename: str,
+        find_line_start: bool,
+    ) -> LogInfo:
+        assert type(filename) is str
+        assert type(find_line_start) is bool
+        assert len(filename) > 0
+        assert os_ops is not None
+        assert isinstance(os_ops, OsOperations)
+
+        file_size = os_ops.get_file_size(filename)
+        assert type(file_size) is int
+        assert file_size >= 0
+
+        if not find_line_start:
+            return __class__.LogInfo(
+                position=file_size,
+                tail=b'',
+            )
+
+        tail = internal_utils.read_line_to_pos__bin(
+            os_ops,
+            filename,
+            file_size,
+        )
+
+        return __class__.LogInfo(
+            position=file_size,
+            tail=tail,
+        )
+
 
 class PostgresNodeUtils:
     @staticmethod
-    def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:
+    def detect_port_conflict(log_reader: PostgresNodeLogReader) -> bool:
         assert type(log_reader) is PostgresNodeLogReader
 
         blocks = log_reader.read()

{testgres-1.15.0 → testgres-1.15.2/testgres.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: testgres
-Version: 1.15.0
+Version: 1.15.2
 Summary: Testing utility for PostgreSQL and its extensions
 Author-email: Postgres Professional <testgres@postgrespro.ru>
 License: PostgreSQL
@@ -27,7 +27,7 @@ Requires-Dist: port-for>=0.4
 Requires-Dist: six>=1.9.0
 Requires-Dist: psutil
 Requires-Dist: packaging
-Requires-Dist: testgres.os_ops<4.0.0,>=3.1.0
+Requires-Dist: testgres.os_ops<4.0.0,>=3.2.0
 Dynamic: license-file
 
 [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/ci.yml)

{testgres-1.15.0 → testgres-1.15.2}/testgres.egg-info/requires.txt

@@ -3,4 +3,4 @@ port-for>=0.4
 six>=1.9.0
 psutil
 packaging
-testgres.os_ops<4.0.0,>=3.1.0
+testgres.os_ops<4.0.0,>=3.2.0

{testgres-1.15.0 → testgres-1.15.2}/tests/test_os_ops_common.py

@@ -1352,8 +1352,15 @@ class TestOsOpsCommon:
 
         tmpdir = os_ops.get_tempdir()
 
-        filename = os_ops.build_path(
+        filedir = os_ops.build_path(
             tmpdir,
+            "test_isfile_false__not_exist-" + uuid.uuid4().bytes.hex(),
+        )
+
+        os_ops.makedir(filedir)
+
+        filename = os_ops.build_path(
+            filedir,
             name_with_surprize.value,
         )
 
@@ -1369,7 +1376,11 @@ class TestOsOpsCommon:
             pass
         else:
             # We will check a real work with another host
+            assert not os.path.exists(filedir)
             assert not os.path.exists(filename)
+
+            os.mkdir(filedir)
+
             with open(filename, "a"):
                 os.utime(filename, None)
             assert os.path.exists(filename)
@@ -1384,7 +1395,11 @@ class TestOsOpsCommon:
             assert os.path.exists(filename)
             os.remove(filename)
             assert not os.path.exists(filename)
+            os.rmdir(filedir)
+            assert not os.path.exists(filedir)
             logging.info("Local detecter is deleted [{}]".format(filename))
+
+        os_ops.rmdir(filedir)
         return
 
     def test_isfile_false__directory(

{testgres-1.15.0 → testgres-1.15.2}/tests/test_testgres_common.py

@@ -121,7 +121,7 @@ class TestTestgresCommon:
         # Author: Mark G.
         assert v.major == 1
         assert v.minor == 15
-        assert v.micro == 0
+        assert v.micro == 1
 
         assert str(v) == testgres_version
         return
@@ -190,15 +190,15 @@ class TestTestgresCommon:
         return
 
     def test_node_constructor__default(self):
-        node = PostgresNode()
-        assert node._os_ops is not None
-        assert isinstance(node._os_ops, OsOperations)
-        assert node._port_manager is not None
-        assert isinstance(node._port_manager, PortManager)
-        assert node._name is not None
-        assert type(node._name) is str
-        assert node._name != ""
-        assert node._base_dir is None
+        with PostgresNode() as node:
+            assert node._os_ops is not None
+            assert isinstance(node._os_ops, OsOperations)
+            assert node._port_manager is not None
+            assert isinstance(node._port_manager, PortManager)
+            assert node._name is not None
+            assert type(node._name) is str
+            assert node._name != ""
+            assert node._base_dir is None
         return
 
     def test_node_constructor__host(self):
@@ -456,12 +456,70 @@ class TestTestgresCommon:
         assert isinstance(node_svc, PostgresNodeService)
 
         with __class__.helper__get_node(node_svc) as node:
-            node.init().start()
+            node.init()
+
+            nRestartAttempt = 0
+
+            while True:
+                nRestartAttempt += 1
+
+                logging.info("Attempt #{}".format(nRestartAttempt))
+
+                node.start()
+
+                # restart, ok
+                res = node.execute('select 1')
+                assert (res == [(1,)])
+
+                node_log_reader = PostgresNodeLogReader(
+                    node,
+                    from_beginnig=False,
+                )
+
+                try:
+                    node.restart()
+                except StartNodeException as e:
+                    logging.info("Exception ({}): {}".format(
+                        type(e).__name__,
+                        e,
+                    ))
+
+                    if nRestartAttempt == 5:
+                        raise
+
+                    if not PostgresNodeUtils.detect_port_conflict(node_log_reader):
+                        raise
+
+                    logging.info("Node port {} conflicted with another PostgreSQL instance.".format(
+                        node.port
+                    ))
+
+                    logging.info("Wait for node stop")
+
+                    nStopAttemtp = 0
+
+                    while True:
+                        if nStopAttemtp == 5:
+                            raise RuntimeError("Node is not stopped!")
+
+                        nStopAttemtp += 1
+
+                        time.sleep(1)
+
+                        node_status = node.status()
+
+                        logging.info("Node status is {}".format(node_status))
+
+                        if node_status == NodeStatus.Stopped:
+                            break
+                        continue
+
+                    # node is stopped. try again
+                    continue
+
+                assert node.status() == NodeStatus.Running
+                break
 
-            # restart, ok
-            res = node.execute('select 1')
-            assert (res == [(1,)])
-            node.restart()
             res = node.execute('select 2')
             assert (res == [(2,)])
 
@@ -771,6 +829,8 @@ class TestTestgresCommon:
         finally:
             if node.is_started:
                 node.stop()
+
+        node.cleanup(release_resources=True)
         return
 
     def test_kill_backgroud_writer__ok(
@@ -1630,7 +1690,7 @@ class TestTestgresCommon:
 
         nAttempt = 0
         while True:
-            if PostgresNodeUtils.delect_port_conflict(node_log_reader):
+            if PostgresNodeUtils.detect_port_conflict(node_log_reader):
                 logging.info("Node port {} conflicted with another PostgreSQL instance.".format(
                     node.port
                 ))
@@ -1726,12 +1786,9 @@ class TestTestgresCommon:
         __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6")
 
         with __class__.helper__get_node(node_svc) as master:
-            old_version = not __class__.helper__pg_version_ge(current_version, '9.6')
-
             master.init(allow_streaming=True).start()
 
-            if not old_version:
-                master.append_conf('synchronous_commit = remote_apply')
+            master.append_conf('synchronous_commit = remote_apply')
 
             # create standby
             with master.replicate() as standby1, master.replicate() as standby2:
@@ -1748,21 +1805,20 @@ class TestTestgresCommon:
 
                 # set synchronous_standby_names
                 master.set_synchronous_standbys(First(2, [standby1, standby2]))
-                master.restart()
-
-                # the following part of the test is only applicable to newer
-                # versions of PostgresQL
-                if not old_version:
-                    master.safe_psql('create table abc(a int)')
-
-                    # Create a large transaction that will take some time to apply
-                    # on standby to check that it applies synchronously
-                    # (If set synchronous_commit to 'on' or other lower level then
-                    # standby most likely won't catchup so fast and test will fail)
-                    master.safe_psql(
-                        'insert into abc select generate_series(1, 1000000)')
-                    res = standby1.safe_psql('select count(*) from abc')
-                    assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n')
+                master.reload()
+
+                master.safe_psql('create table abc(a int)')
+
+                # Create a large transaction that will take some time to apply
+                # on standby to check that it applies synchronously
+                # (If set synchronous_commit to 'on' or other lower level then
+                # standby most likely won't catchup so fast and test will fail)
+                master.safe_psql(
+                    'insert into abc select generate_series(1, 1000000)',
+                )
+                res = standby1.safe_psql('select count(*) from abc')
+                assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n')
+        return
 
     def test_logical_replication(self, node_svc: PostgresNodeService):
         assert isinstance(node_svc, PostgresNodeService)

{testgres-1.15.0 → testgres-1.15.2}/tests/test_testgres_remote.py

@@ -82,6 +82,7 @@ class TestTestgresRemote:
         assert isinstance(node_version, Version)
 
         if node.version < Version("11"):
+            node.cleanup(release_resources=True)
             pytest.skip("This test does not work on old PG10-.")
 
         try:
@@ -152,6 +153,8 @@ class TestTestgresRemote:
                 assert expectedMsg2 in exc.error
                 continue
 
+            node.cleanup(release_resources=True)
+
             if not errorIsDetected:
                 pytest.xfail("All the bad data are processed without errors!")
 

testgres-1.15.0/src/impl/internal_utils.py

@@ -1,20 +0,0 @@
-import logging
-
-
-def send_log(level: int, msg: str) -> None:
-    assert type(level) is int
-    assert type(msg) is str
-
-    return logging.log(level, "[testgres] " + msg)
-
-
-def send_log_info(msg: str) -> None:
-    assert type(msg) is str
-
-    return send_log(logging.INFO, msg)
-
-
-def send_log_debug(msg: str) -> None:
-    assert type(msg) is str
-
-    return send_log(logging.DEBUG, msg)