testgres 1.11.0__tar.gz → 1.12.0__tar.gz

testgres-1.12.0/PKG-INFO

@@ -0,0 +1,232 @@
+Metadata-Version: 2.4
+Name: testgres
+Version: 1.12.0
+Summary: Testing utility for PostgreSQL and its extensions
+Home-page: https://github.com/postgrespro/testgres
+Author: Postgres Professional
+Author-email: testgres@postgrespro.ru
+License: PostgreSQL
+Keywords: test,testing,postgresql
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pg8000
+Requires-Dist: port-for>=0.4
+Requires-Dist: six>=1.9.0
+Requires-Dist: psutil
+Requires-Dist: packaging
+Requires-Dist: testgres.os_ops<1.0.0,>=0.0.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: summary
+
+[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres)
+[![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)
+[![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres)
+
+[Documentation](https://postgrespro.github.io/testgres/)
+
+# testgres
+
+Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer.
+
+## Installation
+
+Install `testgres` from PyPI:
+
+```sh
+pip install testgres
+```
+
+Use a dedicated virtual environment for isolated test dependencies.
+
+## Usage
+
+### Environment
+
+> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`.
+
+Specify a custom PostgreSQL installation in one of the following ways:
+
+- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable.
+- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries.
+
+Example:
+
+```sh
+export PG_BIN=$HOME/pg_16/bin
+python my_tests.py
+```
+
+### Examples
+
+Create a temporary node, run queries, and let `testgres` clean up automatically:
+
+```python
+# create a node with a random name, port, and data directory
+with testgres.get_new_node() as node:
+
+    # run initdb
+    node.init()
+
+    # start PostgreSQL
+    node.start()
+
+    # execute a query in the default database
+    print(node.execute('select 1'))
+
+# the node is stopped and its files are removed automatically
+```
+
+### Query helpers
+
+`testgres` provides four helpers for executing queries against the node:
+
+| Command | Description |
+|---------|-------------|
+| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. |
+| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. |
+| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. |
+| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. |
+
+Example of transactional usage:
+
+```python
+with node.connect() as con:
+    con.begin('serializable')
+    print(con.execute('select %s', 1))
+    con.rollback()
+```
+
+### Logging
+
+By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection.
+
+> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically.
+
+`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes:
+
+```python
+import logging
+
+# write everything to /tmp/testgres.log
+logging.basicConfig(filename='/tmp/testgres.log')
+
+# enable logging and create two nodes
+testgres.configure_testgres(use_python_logging=True)
+node1 = testgres.get_new_node().init().start()
+node2 = testgres.get_new_node().init().start()
+
+node1.execute('select 1')
+node2.execute('select 2')
+
+# disable logging
+testgres.configure_testgres(use_python_logging=False)
+```
+
+See `tests/test_simple.py` for a complete logging example.
+
+### Backup and replication
+
+Creating backups and spawning replicas is straightforward:
+
+```python
+with testgres.get_new_node('master') as master:
+    master.init().start()
+
+    with master.backup() as backup:
+        replica = backup.spawn_replica('replica').start()
+        replica.catchup()
+
+        print(replica.execute('postgres', 'select 1'))
+```
+
+### Benchmarks
+
+Use `pgbench` through `testgres` to run quick benchmarks:
+
+```python
+with testgres.get_new_node('master') as master:
+    master.init().start()
+
+    result = master.pgbench_init(scale=2).pgbench_run(time=10)
+    print(result)
+```
+
+### Custom configuration
+
+`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`:
+
+```python
+extra_conf = "shared_preload_libraries = 'postgres_fdw'"
+
+with testgres.get_new_node().init() as master:
+    master.default_conf(fsync=True, allow_streaming=True)
+    master.append_conf('postgresql.conf', extra_conf)
+```
+
+`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines.
+
+### Remote mode
+
+You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration:
+
+```python
+from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node
+
+conn_params = ConnectionParams(
+    host='example.com',
+    username='postgres',
+    ssh_key='/path/to/ssh/key'
+)
+os_ops = RemoteOperations(conn_params)
+
+TestgresConfig.set_os_ops(os_ops=os_ops)
+
+def test_basic_query():
+    with get_remote_node(conn_params=conn_params) as node:
+        node.init().start()
+        assert node.execute('SELECT 1') == [(1,)]
+```
+
+### Pytest integration
+
+Use fixtures to create and clean up nodes automatically when testing with `pytest`:
+
+```python
+import pytest
+import testgres
+
+@pytest.fixture
+def pg_node():
+    node = testgres.get_new_node().init().start()
+    try:
+        yield node
+    finally:
+        node.stop()
+        node.cleanup()
+
+def test_simple(pg_node):
+    assert pg_node.execute('select 1')[0][0] == 1
+```
+
+This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails.
+
+### Scaling tips
+
+- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`.
+- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.
+- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results.
+
+## Authors
+
+[Ildar Musin](https://github.com/zilder)  
+[Dmitry Ivanov](https://github.com/funbringer)  
+[Ildus Kurbangaliev](https://github.com/ildus)  
+[Yury Zhuravlev](https://github.com/stalkerg)

testgres-1.12.0/README.md

@@ -0,0 +1,204 @@
+[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres)
+[![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres)
+[![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres)
+
+[Documentation](https://postgrespro.github.io/testgres/)
+
+# testgres
+
+Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer.
+
+## Installation
+
+Install `testgres` from PyPI:
+
+```sh
+pip install testgres
+```
+
+Use a dedicated virtual environment for isolated test dependencies.
+
+## Usage
+
+### Environment
+
+> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`.
+
+Specify a custom PostgreSQL installation in one of the following ways:
+
+- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable.
+- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries.
+
+Example:
+
+```sh
+export PG_BIN=$HOME/pg_16/bin
+python my_tests.py
+```
+
+### Examples
+
+Create a temporary node, run queries, and let `testgres` clean up automatically:
+
+```python
+# create a node with a random name, port, and data directory
+with testgres.get_new_node() as node:
+
+    # run initdb
+    node.init()
+
+    # start PostgreSQL
+    node.start()
+
+    # execute a query in the default database
+    print(node.execute('select 1'))
+
+# the node is stopped and its files are removed automatically
+```
+
+### Query helpers
+
+`testgres` provides four helpers for executing queries against the node:
+
+| Command | Description |
+|---------|-------------|
+| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. |
+| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. |
+| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. |
+| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. |
+
+Example of transactional usage:
+
+```python
+with node.connect() as con:
+    con.begin('serializable')
+    print(con.execute('select %s', 1))
+    con.rollback()
+```
+
+### Logging
+
+By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection.
+
+> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically.
+
+`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes:
+
+```python
+import logging
+
+# write everything to /tmp/testgres.log
+logging.basicConfig(filename='/tmp/testgres.log')
+
+# enable logging and create two nodes
+testgres.configure_testgres(use_python_logging=True)
+node1 = testgres.get_new_node().init().start()
+node2 = testgres.get_new_node().init().start()
+
+node1.execute('select 1')
+node2.execute('select 2')
+
+# disable logging
+testgres.configure_testgres(use_python_logging=False)
+```
+
+See `tests/test_simple.py` for a complete logging example.
+
+### Backup and replication
+
+Creating backups and spawning replicas is straightforward:
+
+```python
+with testgres.get_new_node('master') as master:
+    master.init().start()
+
+    with master.backup() as backup:
+        replica = backup.spawn_replica('replica').start()
+        replica.catchup()
+
+        print(replica.execute('postgres', 'select 1'))
+```
+
+### Benchmarks
+
+Use `pgbench` through `testgres` to run quick benchmarks:
+
+```python
+with testgres.get_new_node('master') as master:
+    master.init().start()
+
+    result = master.pgbench_init(scale=2).pgbench_run(time=10)
+    print(result)
+```
+
+### Custom configuration
+
+`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`:
+
+```python
+extra_conf = "shared_preload_libraries = 'postgres_fdw'"
+
+with testgres.get_new_node().init() as master:
+    master.default_conf(fsync=True, allow_streaming=True)
+    master.append_conf('postgresql.conf', extra_conf)
+```
+
+`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines.
+
+### Remote mode
+
+You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration:
+
+```python
+from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node
+
+conn_params = ConnectionParams(
+    host='example.com',
+    username='postgres',
+    ssh_key='/path/to/ssh/key'
+)
+os_ops = RemoteOperations(conn_params)
+
+TestgresConfig.set_os_ops(os_ops=os_ops)
+
+def test_basic_query():
+    with get_remote_node(conn_params=conn_params) as node:
+        node.init().start()
+        assert node.execute('SELECT 1') == [(1,)]
+```
+
+### Pytest integration
+
+Use fixtures to create and clean up nodes automatically when testing with `pytest`:
+
+```python
+import pytest
+import testgres
+
+@pytest.fixture
+def pg_node():
+    node = testgres.get_new_node().init().start()
+    try:
+        yield node
+    finally:
+        node.stop()
+        node.cleanup()
+
+def test_simple(pg_node):
+    assert pg_node.execute('select 1')[0][0] == 1
+```
+
+This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails.
+
+### Scaling tips
+
+- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`.
+- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories.
+- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results.
+
+## Authors
+
+[Ildar Musin](https://github.com/zilder)  
+[Dmitry Ivanov](https://github.com/funbringer)  
+[Ildus Kurbangaliev](https://github.com/ildus)  
+[Yury Zhuravlev](https://github.com/stalkerg)

{testgres-1.11.0 → testgres-1.12.0}/setup.py

@@ -11,7 +11,8 @@ install_requires = [
     "port-for>=0.4",
     "six>=1.9.0",
     "psutil",
-    "packaging"
+    "packaging",
+    "testgres.os_ops>=0.0.2,<1.0.0"
 ]
 
 # Add compatibility enum class
@@ -27,9 +28,10 @@ with open('README.md', 'r') as f:
     readme = f.read()
 
 setup(
-    version='1.11.0',
+    version='1.12.0',
     name='testgres',
-    packages=['testgres', 'testgres.operations'],
+    packages=['testgres', 'testgres.impl'],
+    package_dir={"testgres": "src"},
     description='Testing utility for PostgreSQL and its extensions',
     url='https://github.com/postgrespro/testgres',
     long_description=readme,

{testgres-1.11.0/testgres → testgres-1.12.0/src}/__init__.py

@@ -33,8 +33,9 @@ from .enums import \
     ProcessType, \
     DumpFormat
 
-from .node import PostgresNode, NodeApp
+from .node import PostgresNode
 from .node import PortManager
+from .node_app import NodeApp
 
 from .utils import \
     reserve_port, \
@@ -50,9 +51,9 @@ from .standby import \
 
 from .config import testgres_config
 
-from .operations.os_ops import OsOperations, ConnectionParams
-from .operations.local_ops import LocalOperations
-from .operations.remote_ops import RemoteOperations
+from testgres.operations.os_ops import OsOperations, ConnectionParams
+from testgres.operations.local_ops import LocalOperations
+from testgres.operations.remote_ops import RemoteOperations
 
 __all__ = [
     "get_new_node",
@@ -62,8 +63,9 @@ __all__ = [
     "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError",
     "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException",
     "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat",
-    "PostgresNode", "NodeApp",
-    "PortManager",
+    NodeApp.__name__,
+    PostgresNode.__name__,
+    PortManager.__name__,
     "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version",
     "First", "Any",
     "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams"

{testgres-1.11.0/testgres → testgres-1.12.0/src}/backup.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-import os
-
 from six import raise_from
 
 from .enums import XLogMethod
@@ -15,7 +13,7 @@ from .consts import \
 
 from .exceptions import BackupException
 
-from .operations.os_ops import OsOperations
+from testgres.operations.os_ops import OsOperations
 
 from .utils import \
     get_bin_path2, \
@@ -29,7 +27,9 @@ class NodeBackup(object):
     """
     @property
     def log_file(self):
-        return os.path.join(self.base_dir, BACKUP_LOG_FILE)
+        assert self.os_ops is not None
+        assert isinstance(self.os_ops, OsOperations)
+        return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE)
 
     def __init__(self,
                  node,
@@ -75,7 +75,7 @@ class NodeBackup(object):
         # private
         self._available = True
 
-        data_dir = os.path.join(self.base_dir, DATA_DIR)
+        data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR)
 
         _params = [
             get_bin_path2(self.os_ops, "pg_basebackup"),
@@ -112,10 +112,13 @@ class NodeBackup(object):
         available = not destroy
 
         if available:
+            assert self.os_ops is not None
+            assert isinstance(self.os_ops, OsOperations)
+
             dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE)
 
-            data1 = os.path.join(self.base_dir, DATA_DIR)
-            data2 = os.path.join(dest_base_dir, DATA_DIR)
+            data1 = self.os_ops.build_path(self.base_dir, DATA_DIR)
+            data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR)
 
             try:
                 # Copy backup to new data dir
@@ -160,10 +163,6 @@ class NodeBackup(object):
         assert type(node) == self.original_node.__class__  # noqa: E721
 
         with clean_on_error(node) as node:
-
-            # New nodes should always remove dir tree
-            node._should_rm_dirs = True
-
             # Set a new port
             node.append_conf(filename=PG_CONF_FILE, line='\n')
             node.append_conf(filename=PG_CONF_FILE, port=node.port)
@@ -184,14 +183,19 @@ class NodeBackup(object):
         """
 
         # Build a new PostgresNode
-        with clean_on_error(self.spawn_primary(name=name,
-                                               destroy=destroy)) as node:
+        node = self.spawn_primary(name=name, destroy=destroy)
+        assert node is not None
 
+        try:
             # Assign it a master and a recovery file (private magic)
             node._assign_master(self.original_node)
             node._create_recovery_conf(username=self.username, slot=slot)
+        except:  # noqa: E722
+            # TODO: Pass 'final=True' ?
+            node.cleanup(release_resources=True)
+            raise
 
-            return node
+        return node
 
     def cleanup(self):
         """

{testgres-1.11.0/testgres → testgres-1.12.0/src}/cache.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 
-import os
-
 from six import raise_from
 
 from .config import testgres_config
@@ -18,16 +16,20 @@ from .utils import \
     get_bin_path2, \
     execute_utility2
 
-from .operations.local_ops import LocalOperations
-from .operations.os_ops import OsOperations
+from testgres.operations.local_ops import LocalOperations
+from testgres.operations.os_ops import OsOperations
 
 
-def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True):
+def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True):
     """
     Perform initdb or use cached node files.
     """
 
-    assert os_ops is not None
+    assert os_ops is None or isinstance(os_ops, OsOperations)
+
+    if os_ops is None:
+        os_ops = LocalOperations.get_single_instance()
+
     assert isinstance(os_ops, OsOperations)
 
     def make_utility_path(name):
@@ -35,7 +37,7 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo
         assert type(name) == str  # noqa: E721
 
         if bin_path:
-            return os.path.join(bin_path, name)
+            return os_ops.build_path(bin_path, name)
 
         return get_bin_path2(os_ops, name)
 
@@ -68,7 +70,7 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo
                 # XXX: write new unique system id to control file
                 # Some users might rely upon unique system ids, but
                 # our initdb caching mechanism breaks this contract.
-                pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE)
+                pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE)
                 system_id = generate_system_id()
                 cur_pg_control = os_ops.read(pg_control, binary=True)
                 new_pg_control = system_id + cur_pg_control[len(system_id):]

{testgres-1.11.0/testgres → testgres-1.12.0/src}/config.py

@@ -9,8 +9,8 @@ import tempfile
 from contextlib import contextmanager
 
 from .consts import TMP_CACHE
-from .operations.os_ops import OsOperations
-from .operations.local_ops import LocalOperations
+from testgres.operations.os_ops import OsOperations
+from testgres.operations.local_ops import LocalOperations
 
 log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper()
 log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s')
@@ -50,8 +50,9 @@ class GlobalConfig(object):
     _cached_initdb_dir = None
     """ underlying class attribute for cached_initdb_dir property """
 
-    os_ops = LocalOperations()
+    os_ops = LocalOperations.get_single_instance()
     """ OsOperation object that allows work on remote host """
+
     @property
     def cached_initdb_dir(self):
         """ path to a temp directory for cached initdb. """