testgres 1.10.5__tar.gz → 1.11.1__tar.gz

{testgres-1.10.5/testgres.egg-info → testgres-1.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: testgres
-Version: 1.10.5
+Version: 1.11.1
 Summary: Testing utility for PostgreSQL and its extensions
 Home-page: https://github.com/postgrespro/testgres
 Author: Postgres Professional
@@ -21,10 +21,11 @@ Dynamic: description-content-type
 Dynamic: home-page
 Dynamic: keywords
 Dynamic: license
+Dynamic: license-file
 Dynamic: requires-dist
 Dynamic: summary
 
-[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches)
+[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres)
 [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)
 [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres)
 
@@ -32,7 +33,7 @@ Dynamic: summary
 
 # testgres
 
-PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported.
+PostgreSQL testing utility. Python 3.7.17+ is supported.
 
 
 ## Installation

{testgres-1.10.5 → testgres-1.11.1}/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches)
+[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres)
 [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)
 [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres)
 
@@ -6,7 +6,7 @@
 
 # testgres
 
-PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported.
+PostgreSQL testing utility. Python 3.7.17+ is supported.
 
 
 ## Installation

{testgres-1.10.5 → testgres-1.11.1}/setup.py

@@ -27,9 +27,9 @@ with open('README.md', 'r') as f:
     readme = f.read()
 
 setup(
-    version='1.10.5',
+    version='1.11.1',
     name='testgres',
-    packages=['testgres', 'testgres.operations', 'testgres.helpers'],
+    packages=['testgres', 'testgres.operations', 'testgres.impl'],
     description='Testing utility for PostgreSQL and its extensions',
     url='https://github.com/postgrespro/testgres',
     long_description=readme,

{testgres-1.10.5 → testgres-1.11.1}/testgres/__init__.py

@@ -33,7 +33,9 @@ from .enums import \
     ProcessType, \
     DumpFormat
 
-from .node import PostgresNode, NodeApp
+from .node import PostgresNode
+from .node import PortManager
+from .node_app import NodeApp
 
 from .utils import \
     reserve_port, \
@@ -53,8 +55,6 @@ from .operations.os_ops import OsOperations, ConnectionParams
 from .operations.local_ops import LocalOperations
 from .operations.remote_ops import RemoteOperations
 
-from .helpers.port_manager import PortManager
-
 __all__ = [
     "get_new_node",
     "get_remote_node",
@@ -63,8 +63,10 @@ __all__ = [
     "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
     "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
     "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
-    "PostgresNode", "NodeApp",
+    NodeApp.__name__,
+    PostgresNode.__name__,
+    PortManager.__name__,
     "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",
-    "First", "Any", "PortManager",
+    "First", "Any",
     "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"
 ]

{testgres-1.10.5 → testgres-1.11.1}/testgres/api.py

@@ -47,8 +47,6 @@ def get_remote_node(name=None, conn_params=None):
     Simply a wrapper around :class:`.PostgresNode` constructor for remote node.
     See :meth:`.PostgresNode.__init__` for details.
     For remote connection you can add the next parameter:
-    conn_params = ConnectionParams(host='127.0.0.1',
-                                   ssh_key=None,
-                                   username=default_username())
+    conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username())
     """
     return get_new_node(name=name, conn_params=conn_params)

{testgres-1.10.5 → testgres-1.11.1}/testgres/backup.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-import os
-
 from six import raise_from
 
 from .enums import XLogMethod
@@ -15,9 +13,11 @@ from .consts import \
 
 from .exceptions import BackupException
 
+from .operations.os_ops import OsOperations
+
 from .utils import \
-    get_bin_path, \
-    execute_utility, \
+    get_bin_path2, \
+    execute_utility2, \
     clean_on_error
 
 
@@ -27,7 +27,9 @@ class NodeBackup(object):
     """
     @property
     def log_file(self):
-        return os.path.join(self.base_dir, BACKUP_LOG_FILE)
+        assert self.os_ops is not None
+        assert isinstance(self.os_ops, OsOperations)
+        return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE)
 
     def __init__(self,
                  node,
@@ -44,6 +46,9 @@ class NodeBackup(object):
             username: database user name.
             xlog_method: none | fetch | stream (see docs)
         """
+        assert node.os_ops is not None
+        assert isinstance(node.os_ops, OsOperations)
+
         if not options:
             options = []
         self.os_ops = node.os_ops
@@ -70,10 +75,10 @@ class NodeBackup(object):
         # private
         self._available = True
 
-        data_dir = os.path.join(self.base_dir, DATA_DIR)
+        data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR)
 
         _params = [
-            get_bin_path("pg_basebackup"),
+            get_bin_path2(self.os_ops, "pg_basebackup"),
             "-p", str(node.port),
             "-h", node.host,
             "-U", username,
@@ -81,7 +86,7 @@ class NodeBackup(object):
             "-X", xlog_method.value
         ]  # yapf: disable
         _params += options
-        execute_utility(_params, self.log_file)
+        execute_utility2(self.os_ops, _params, self.log_file)
 
     def __enter__(self):
         return self
@@ -107,10 +112,13 @@ class NodeBackup(object):
         available = not destroy
 
         if available:
+            assert self.os_ops is not None
+            assert isinstance(self.os_ops, OsOperations)
+
             dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE)
 
-            data1 = os.path.join(self.base_dir, DATA_DIR)
-            data2 = os.path.join(dest_base_dir, DATA_DIR)
+            data1 = self.os_ops.build_path(self.base_dir, DATA_DIR)
+            data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR)
 
             try:
                 # Copy backup to new data dir
@@ -142,12 +150,19 @@ class NodeBackup(object):
         base_dir = self._prepare_dir(destroy)
 
         # Build a new PostgresNode
-        NodeClass = self.original_node.__class__
-        with clean_on_error(NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)) as node:
+        assert self.original_node is not None
 
-            # New nodes should always remove dir tree
-            node._should_rm_dirs = True
+        if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")):
+            node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir)
+        else:
+            # For backward compatibility
+            NodeClass = self.original_node.__class__
+            node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)
+
+        assert node is not None
+        assert type(node) == self.original_node.__class__  # noqa: E721
 
+        with clean_on_error(node) as node:
             # Set a new port
             node.append_conf(filename=PG_CONF_FILE, line='\n')
             node.append_conf(filename=PG_CONF_FILE, port=node.port)
@@ -168,14 +183,19 @@ class NodeBackup(object):
         """
 
         # Build a new PostgresNode
-        with clean_on_error(self.spawn_primary(name=name,
-                                               destroy=destroy)) as node:
+        node = self.spawn_primary(name=name, destroy=destroy)
+        assert node is not None
 
+        try:
             # Assign it a master and a recovery file (private magic)
             node._assign_master(self.original_node)
             node._create_recovery_conf(username=self.username, slot=slot)
+        except:  # noqa: E722
+            # TODO: Pass 'final=True' ?
+            node.cleanup(release_resources=True)
+            raise
 
-            return node
+        return node
 
     def cleanup(self):
         """

{testgres-1.10.5 → testgres-1.11.1}/testgres/cache.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-import os
-
 from six import raise_from
 
 from .config import testgres_config
@@ -15,23 +13,39 @@ from .exceptions import \
     ExecUtilException
 
 from .utils import \
-    get_bin_path, \
-    execute_utility
+    get_bin_path2, \
+    execute_utility2
 
 from .operations.local_ops import LocalOperations
 from .operations.os_ops import OsOperations
 
 
-def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True):
+def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True):
     """
     Perform initdb or use cached node files.
     """
 
+    assert os_ops is None or isinstance(os_ops, OsOperations)
+
+    if os_ops is None:
+        os_ops = LocalOperations.get_single_instance()
+
+    assert isinstance(os_ops, OsOperations)
+
+    def make_utility_path(name):
+        assert name is not None
+        assert type(name) == str  # noqa: E721
+
+        if bin_path:
+            return os_ops.build_path(bin_path, name)
+
+        return get_bin_path2(os_ops, name)
+
     def call_initdb(initdb_dir, log=logfile):
         try:
-            initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb")
+            initdb_path = make_utility_path("initdb")
             _params = [initdb_path, "-D", initdb_dir, "-N"]
-            execute_utility(_params + (params or []), log)
+            execute_utility2(os_ops, _params + (params or []), log)
         except ExecUtilException as e:
             raise_from(InitNodeException("Failed to run initdb"), e)
 
@@ -56,15 +70,15 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo
                 # XXX: write new unique system id to control file
                 # Some users might rely upon unique system ids, but
                 # our initdb caching mechanism breaks this contract.
-                pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE)
+                pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE)
                 system_id = generate_system_id()
                 cur_pg_control = os_ops.read(pg_control, binary=True)
                 new_pg_control = system_id + cur_pg_control[len(system_id):]
                 os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True)
 
                 # XXX: build new WAL segment with our system id
-                _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"]
-                execute_utility(_params, logfile)
+                _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"]
+                execute_utility2(os_ops, _params, logfile)
 
         except ExecUtilException as e:
             msg = "Failed to reset WAL for system id"

{testgres-1.10.5 → testgres-1.11.1}/testgres/config.py

@@ -50,8 +50,9 @@ class GlobalConfig(object):
     _cached_initdb_dir = None
     """ underlying class attribute for cached_initdb_dir property """
 
-    os_ops = LocalOperations()
+    os_ops = LocalOperations.get_single_instance()
     """ OsOperation object that allows work on remote host """
+
     @property
     def cached_initdb_dir(self):
         """ path to a temp directory for cached initdb. """

{testgres-1.10.5 → testgres-1.11.1}/testgres/connection.py

@@ -42,11 +42,13 @@ class NodeConnection(object):
 
         self._node = node
 
-        self._connection = node.os_ops.db_connect(dbname=dbname,
-                                                  user=username,
-                                                  password=password,
-                                                  host=node.host,
-                                                  port=node.port)
+        self._connection = pglib.connect(
+            database=dbname,
+            user=username,
+            password=password,
+            host=node.host,
+            port=node.port
+        )
 
         self._connection.autocommit = autocommit
         self._cursor = self.connection.cursor()

{testgres-1.10.5 → testgres-1.11.1}/testgres/consts.py

@@ -35,3 +35,7 @@ MAX_WAL_SENDERS = 10
 
 # logical replication settings
 LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60
+
+PG_CTL__STATUS__OK = 0
+PG_CTL__STATUS__NODE_IS_STOPPED = 3
+PG_CTL__STATUS__BAD_DATADIR = 4

{testgres-1.10.5 → testgres-1.11.1}/testgres/exceptions.py

@@ -7,6 +7,10 @@ class TestgresException(Exception):
     pass
 
 
+class PortForException(TestgresException):
+    pass
+
+
 @six.python_2_unicode_compatible
 class ExecUtilException(TestgresException):
     def __init__(self, message=None, command=None, exit_code=0, out=None, error=None):

testgres-1.11.1/testgres/impl/port_manager__generic.py

@@ -0,0 +1,97 @@
+from ..operations.os_ops import OsOperations
+
+from ..port_manager import PortManager
+from ..exceptions import PortForException
+
+import threading
+import random
+import typing
+import logging
+
+
+class PortManager__Generic(PortManager):
+    _C_MIN_PORT_NUMBER = 1024
+    _C_MAX_PORT_NUMBER = 65535
+
+    _os_ops: OsOperations
+    _guard: object
+    # TODO: is there better to use bitmap fot _available_ports?
+    _available_ports: typing.Set[int]
+    _reserved_ports: typing.Set[int]
+
+    def __init__(self, os_ops: OsOperations):
+        assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER
+
+        assert os_ops is not None
+        assert isinstance(os_ops, OsOperations)
+        self._os_ops = os_ops
+        self._guard = threading.Lock()
+
+        self._available_ports = set(
+            range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1)
+        )
+        assert len(self._available_ports) == (
+            (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1
+        )
+        self._reserved_ports = set()
+        return
+
+    def reserve_port(self) -> int:
+        assert self._guard is not None
+        assert type(self._available_ports) == set  # noqa: E721t
+        assert type(self._reserved_ports) == set  # noqa: E721
+
+        with self._guard:
+            t = tuple(self._available_ports)
+            assert len(t) == len(self._available_ports)
+            sampled_ports = random.sample(t, min(len(t), 100))
+            t = None
+
+            for port in sampled_ports:
+                assert type(port) == int  # noqa: E721
+                assert not (port in self._reserved_ports)
+                assert port in self._available_ports
+
+                assert port >= __class__._C_MIN_PORT_NUMBER
+                assert port <= __class__._C_MAX_PORT_NUMBER
+
+                if not self._os_ops.is_port_free(port):
+                    continue
+
+                self._reserved_ports.add(port)
+                self._available_ports.discard(port)
+                assert port in self._reserved_ports
+                assert not (port in self._available_ports)
+                __class__.helper__send_debug_msg("Port {} is reserved.", port)
+                return port
+
+        raise PortForException("Can't select a port.")
+
+    def release_port(self, number: int) -> None:
+        assert type(number) == int  # noqa: E721
+        assert number >= __class__._C_MIN_PORT_NUMBER
+        assert number <= __class__._C_MAX_PORT_NUMBER
+
+        assert self._guard is not None
+        assert type(self._reserved_ports) == set  # noqa: E721
+
+        with self._guard:
+            assert number in self._reserved_ports
+            assert not (number in self._available_ports)
+            self._available_ports.add(number)
+            self._reserved_ports.discard(number)
+            assert not (number in self._reserved_ports)
+            assert number in self._available_ports
+            __class__.helper__send_debug_msg("Port {} is released.", number)
+        return
+
+    @staticmethod
+    def helper__send_debug_msg(msg_template: str, *args) -> None:
+        assert msg_template is not None
+        assert args is not None
+        assert type(msg_template) == str  # noqa: E721
+        assert type(args) == tuple  # noqa: E721
+        assert msg_template != ""
+        s = "[port manager] "
+        s += msg_template.format(*args)
+        logging.debug(s)

testgres-1.11.1/testgres/impl/port_manager__this_host.py

@@ -0,0 +1,33 @@
+from ..port_manager import PortManager
+
+from .. import utils
+
+import threading
+
+
+class PortManager__ThisHost(PortManager):
+    sm_single_instance: PortManager = None
+    sm_single_instance_guard = threading.Lock()
+
+    @staticmethod
+    def get_single_instance() -> PortManager:
+        assert __class__ == PortManager__ThisHost
+        assert __class__.sm_single_instance_guard is not None
+
+        if __class__.sm_single_instance is not None:
+            assert type(__class__.sm_single_instance) == __class__  # noqa: E721
+            return __class__.sm_single_instance
+
+        with __class__.sm_single_instance_guard:
+            if __class__.sm_single_instance is None:
+                __class__.sm_single_instance = __class__()
+        assert __class__.sm_single_instance is not None
+        assert type(__class__.sm_single_instance) == __class__  # noqa: E721
+        return __class__.sm_single_instance
+
+    def reserve_port(self) -> int:
+        return utils.reserve_port()
+
+    def release_port(self, number: int) -> None:
+        assert type(number) == int  # noqa: E721
+        return utils.release_port(number)