pgmonkey 2.1.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgmonkey-2.1.0/src/pgmonkey.egg-info → pgmonkey-2.3.0}/PKG-INFO +54 -5
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/README.md +53 -4
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/pyproject.toml +1 -1
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_pg_server_config_subparser.py +6 -1
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_pgconfig_subparser.py +6 -1
- pgmonkey-2.3.0/src/pgmonkey/common/templates/postgres.yaml +66 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/postgres/async_connection.py +21 -13
- pgmonkey-2.3.0/src/pgmonkey/connections/postgres/async_pool_connection.py +143 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/postgres/normal_connection.py +27 -13
- pgmonkey-2.3.0/src/pgmonkey/connections/postgres/pool_connection.py +151 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/postgres/postgres_connection_factory.py +40 -8
- pgmonkey-2.3.0/src/pgmonkey/managers/pg_server_config_manager.py +57 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/pgcodegen_manager.py +5 -2
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/pgconnection_manager.py +36 -4
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/serversettings/postgres_server_config_generator.py +92 -3
- pgmonkey-2.3.0/src/pgmonkey/serversettings/postgres_server_settings_inspector.py +155 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/conftest.py +8 -1
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_async_connection.py +174 -0
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_async_pool_connection.py +292 -0
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_code_generator.py +170 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_connection_caching.py +23 -0
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_connection_factory.py +179 -0
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_normal_connection.py +275 -0
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_pool_connection.py +275 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_server_config_generator.py +146 -3
- pgmonkey-2.3.0/src/pgmonkey/tests/unit/test_server_settings_inspector.py +243 -0
- pgmonkey-2.3.0/src/pgmonkey/tools/connection_code_generator.py +320 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tools/csv_data_exporter.py +25 -13
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tools/csv_data_importer.py +50 -25
- {pgmonkey-2.1.0 → pgmonkey-2.3.0/src/pgmonkey.egg-info}/PKG-INFO +54 -5
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey.egg-info/SOURCES.txt +2 -0
- pgmonkey-2.1.0/src/pgmonkey/common/templates/postgres.yaml +0 -44
- pgmonkey-2.1.0/src/pgmonkey/connections/postgres/async_pool_connection.py +0 -108
- pgmonkey-2.1.0/src/pgmonkey/connections/postgres/pool_connection.py +0 -108
- pgmonkey-2.1.0/src/pgmonkey/managers/pg_server_config_manager.py +0 -22
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_async_connection.py +0 -98
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_async_pool_connection.py +0 -77
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_code_generator.py +0 -64
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_connection_factory.py +0 -77
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_normal_connection.py +0 -148
- pgmonkey-2.1.0/src/pgmonkey/tests/unit/test_pool_connection.py +0 -97
- pgmonkey-2.1.0/src/pgmonkey/tools/connection_code_generator.py +0 -141
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/LICENSE +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/NOTICE +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/setup.cfg +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_export_subparser.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_import_subparser.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_settings_subparser.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/cli/cli_toplevel_parser.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/common/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/common/config/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/common/utils/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/common/utils/pathutils.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/base.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/postgres/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/connections/postgres/base_connection.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/pgconfig_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/pgexport_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/pgimport_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/settings_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/managers/toplevel_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/settings/app_settings.yaml +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/integration/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/integration/test_pgconnection_manager_integration.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_base_connection.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_config_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_path_utils.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_pgconnection_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tests/unit/test_settings_manager.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tools/__init__.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey/tools/database_connection_tester.py +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey.egg-info/dependency_links.txt +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey.egg-info/entry_points.txt +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey.egg-info/requires.txt +0 -0
- {pgmonkey-2.1.0 → pgmonkey-2.3.0}/src/pgmonkey.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgmonkey
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: A tool to assist with postgresql database connections
|
|
5
5
|
Author-email: Good Boy <pythonic@rexbytes.com>
|
|
6
6
|
License: MIT
|
|
@@ -52,6 +52,7 @@ Dynamic: license-file
|
|
|
52
52
|
- [Testing a Connection](#testing-a-connection)
|
|
53
53
|
- [Generating Python Code](#generating-python-code)
|
|
54
54
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
55
|
+
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
55
56
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
56
57
|
6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
57
58
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
@@ -138,10 +139,13 @@ postgresql:
|
|
|
138
139
|
pool_settings:
|
|
139
140
|
min_size: 5
|
|
140
141
|
max_size: 20
|
|
142
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
141
143
|
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
142
144
|
max_lifetime: 3600 # Seconds a connection can be reused
|
|
145
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
143
146
|
|
|
144
147
|
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
148
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
145
149
|
async_settings:
|
|
146
150
|
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
147
151
|
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
@@ -152,8 +156,10 @@ postgresql:
|
|
|
152
156
|
async_pool_settings:
|
|
153
157
|
min_size: 5
|
|
154
158
|
max_size: 20
|
|
159
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
155
160
|
max_idle: 300
|
|
156
161
|
max_lifetime: 3600
|
|
162
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
157
163
|
```
|
|
158
164
|
|
|
159
165
|
### Connection Settings
|
|
@@ -184,12 +190,14 @@ Used by `pool` connection type.
|
|
|
184
190
|
|-----------|-------------|---------|
|
|
185
191
|
| `min_size` | Minimum number of connections in the pool | `5` |
|
|
186
192
|
| `max_size` | Maximum number of connections in the pool | `20` |
|
|
193
|
+
| `timeout` | Seconds to wait for a connection from the pool before raising an error | `30` |
|
|
187
194
|
| `max_idle` | Seconds a connection can remain idle before being closed | `300` |
|
|
188
195
|
| `max_lifetime` | Seconds a connection can be reused | `3600` |
|
|
196
|
+
| `check_on_checkout` | Validate connections with `SELECT 1` before handing to caller | `false` |
|
|
189
197
|
|
|
190
198
|
### Async Settings
|
|
191
199
|
|
|
192
|
-
Used by `async` connection
|
|
200
|
+
Used by `async` and `async_pool` connection types. These are applied via SQL `SET` commands when the connection is established. For `async_pool`, they are applied to each connection via a psycopg_pool `configure` callback.
|
|
193
201
|
|
|
194
202
|
| Parameter | Description | Example |
|
|
195
203
|
|-----------|-------------|---------|
|
|
@@ -200,14 +208,16 @@ Used by `async` connection type. These are applied via SQL `SET` commands when t
|
|
|
200
208
|
|
|
201
209
|
### Async Pool Settings
|
|
202
210
|
|
|
203
|
-
Used by `async_pool` connection type. Same parameters as pool settings.
|
|
211
|
+
Used by `async_pool` connection type. Same parameters as pool settings. The `async_settings` section (above) is also applied to async pool connections.
|
|
204
212
|
|
|
205
213
|
| Parameter | Description | Example |
|
|
206
214
|
|-----------|-------------|---------|
|
|
207
215
|
| `min_size` | Minimum connections in the async pool | `5` |
|
|
208
216
|
| `max_size` | Maximum connections in the async pool | `20` |
|
|
217
|
+
| `timeout` | Seconds to wait for a connection from the pool before raising an error | `30` |
|
|
209
218
|
| `max_idle` | Seconds a connection can remain idle | `300` |
|
|
210
219
|
| `max_lifetime` | Seconds a connection can be reused | `3600` |
|
|
220
|
+
| `check_on_checkout` | Validate connections with `SELECT 1` before handing to caller | `false` |
|
|
211
221
|
|
|
212
222
|
## Authentication Methods
|
|
213
223
|
|
|
@@ -309,8 +319,16 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml
|
|
|
309
319
|
|
|
310
320
|
# Generate code for a specific connection type
|
|
311
321
|
pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-type async_pool
|
|
322
|
+
|
|
323
|
+
# Generate code using native psycopg/psycopg_pool instead of pgmonkey
|
|
324
|
+
pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-type pool --library psycopg
|
|
312
325
|
```
|
|
313
326
|
|
|
327
|
+
The `--library` flag controls which library the generated code targets:
|
|
328
|
+
|
|
329
|
+
- `pgmonkey` (default) — generates code using pgmonkey's `PGConnectionManager`.
|
|
330
|
+
- `psycopg` — generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
331
|
+
|
|
314
332
|
### Server Configuration Recommendations
|
|
315
333
|
|
|
316
334
|
Generate recommended PostgreSQL server configuration entries based on your config file:
|
|
@@ -340,6 +358,36 @@ ssl_key_file = 'server.key'
|
|
|
340
358
|
ssl_ca_file = 'ca.crt'
|
|
341
359
|
```
|
|
342
360
|
|
|
361
|
+
### Auditing Live Server Settings
|
|
362
|
+
|
|
363
|
+
Add `--audit` to connect to the live server and compare current settings against recommendations:
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
pgmonkey pgserverconfig --filepath /path/to/config.yaml --audit
|
|
367
|
+
```
|
|
368
|
+
|
|
369
|
+
This queries the server's `pg_settings` (read-only) and displays a comparison table:
|
|
370
|
+
|
|
371
|
+
```
|
|
372
|
+
1) Database type detected: PostgreSQL
|
|
373
|
+
|
|
374
|
+
2) Server settings audit:
|
|
375
|
+
|
|
376
|
+
postgresql.conf:
|
|
377
|
+
|
|
378
|
+
Setting Recommended Current Source Status
|
|
379
|
+
────────────────────────────────────────────────────────────────────
|
|
380
|
+
max_connections 22 100 configuration file OK
|
|
381
|
+
ssl on on configuration file OK
|
|
382
|
+
ssl_cert_file server.crt server.crt configuration file OK
|
|
383
|
+
ssl_key_file server.key server.key configuration file OK
|
|
384
|
+
ssl_ca_file ca.crt ca.crt configuration file OK
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
388
|
+
|
|
389
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified — the audit is entirely read-only.
|
|
390
|
+
|
|
343
391
|
### Importing and Exporting Data
|
|
344
392
|
|
|
345
393
|
**Import data** from a CSV or text file into a PostgreSQL table:
|
|
@@ -493,7 +541,8 @@ pgmonkey handles several production concerns behind the scenes so you don't have
|
|
|
493
541
|
- **Connection caching** — Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
494
542
|
- **Async pool lifecycle** — `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
495
543
|
- **atexit cleanup** — All cached connections are automatically closed when the process exits.
|
|
496
|
-
- **Thread-safe caching** — The connection cache is protected by a threading lock.
|
|
544
|
+
- **Thread-safe caching** — The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
545
|
+
- **Config validation** — Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
497
546
|
|
|
498
547
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
499
548
|
|
|
@@ -725,7 +774,7 @@ Run the tests:
|
|
|
725
774
|
pytest
|
|
726
775
|
```
|
|
727
776
|
|
|
728
|
-
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation
|
|
777
|
+
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, server config generation, and server settings inspection.
|
|
729
778
|
|
|
730
779
|
---
|
|
731
780
|
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
- [Testing a Connection](#testing-a-connection)
|
|
21
21
|
- [Generating Python Code](#generating-python-code)
|
|
22
22
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
23
|
+
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
23
24
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
24
25
|
6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
25
26
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
@@ -106,10 +107,13 @@ postgresql:
|
|
|
106
107
|
pool_settings:
|
|
107
108
|
min_size: 5
|
|
108
109
|
max_size: 20
|
|
110
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
109
111
|
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
110
112
|
max_lifetime: 3600 # Seconds a connection can be reused
|
|
113
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
111
114
|
|
|
112
115
|
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
116
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
113
117
|
async_settings:
|
|
114
118
|
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
115
119
|
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
@@ -120,8 +124,10 @@ postgresql:
|
|
|
120
124
|
async_pool_settings:
|
|
121
125
|
min_size: 5
|
|
122
126
|
max_size: 20
|
|
127
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
123
128
|
max_idle: 300
|
|
124
129
|
max_lifetime: 3600
|
|
130
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
125
131
|
```
|
|
126
132
|
|
|
127
133
|
### Connection Settings
|
|
@@ -152,12 +158,14 @@ Used by `pool` connection type.
|
|
|
152
158
|
|-----------|-------------|---------|
|
|
153
159
|
| `min_size` | Minimum number of connections in the pool | `5` |
|
|
154
160
|
| `max_size` | Maximum number of connections in the pool | `20` |
|
|
161
|
+
| `timeout` | Seconds to wait for a connection from the pool before raising an error | `30` |
|
|
155
162
|
| `max_idle` | Seconds a connection can remain idle before being closed | `300` |
|
|
156
163
|
| `max_lifetime` | Seconds a connection can be reused | `3600` |
|
|
164
|
+
| `check_on_checkout` | Validate connections with `SELECT 1` before handing to caller | `false` |
|
|
157
165
|
|
|
158
166
|
### Async Settings
|
|
159
167
|
|
|
160
|
-
Used by `async` connection
|
|
168
|
+
Used by `async` and `async_pool` connection types. These are applied via SQL `SET` commands when the connection is established. For `async_pool`, they are applied to each connection via a psycopg_pool `configure` callback.
|
|
161
169
|
|
|
162
170
|
| Parameter | Description | Example |
|
|
163
171
|
|-----------|-------------|---------|
|
|
@@ -168,14 +176,16 @@ Used by `async` connection type. These are applied via SQL `SET` commands when t
|
|
|
168
176
|
|
|
169
177
|
### Async Pool Settings
|
|
170
178
|
|
|
171
|
-
Used by `async_pool` connection type. Same parameters as pool settings.
|
|
179
|
+
Used by `async_pool` connection type. Same parameters as pool settings. The `async_settings` section (above) is also applied to async pool connections.
|
|
172
180
|
|
|
173
181
|
| Parameter | Description | Example |
|
|
174
182
|
|-----------|-------------|---------|
|
|
175
183
|
| `min_size` | Minimum connections in the async pool | `5` |
|
|
176
184
|
| `max_size` | Maximum connections in the async pool | `20` |
|
|
185
|
+
| `timeout` | Seconds to wait for a connection from the pool before raising an error | `30` |
|
|
177
186
|
| `max_idle` | Seconds a connection can remain idle | `300` |
|
|
178
187
|
| `max_lifetime` | Seconds a connection can be reused | `3600` |
|
|
188
|
+
| `check_on_checkout` | Validate connections with `SELECT 1` before handing to caller | `false` |
|
|
179
189
|
|
|
180
190
|
## Authentication Methods
|
|
181
191
|
|
|
@@ -277,8 +287,16 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml
|
|
|
277
287
|
|
|
278
288
|
# Generate code for a specific connection type
|
|
279
289
|
pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-type async_pool
|
|
290
|
+
|
|
291
|
+
# Generate code using native psycopg/psycopg_pool instead of pgmonkey
|
|
292
|
+
pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-type pool --library psycopg
|
|
280
293
|
```
|
|
281
294
|
|
|
295
|
+
The `--library` flag controls which library the generated code targets:
|
|
296
|
+
|
|
297
|
+
- `pgmonkey` (default) — generates code using pgmonkey's `PGConnectionManager`.
|
|
298
|
+
- `psycopg` — generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
299
|
+
|
|
282
300
|
### Server Configuration Recommendations
|
|
283
301
|
|
|
284
302
|
Generate recommended PostgreSQL server configuration entries based on your config file:
|
|
@@ -308,6 +326,36 @@ ssl_key_file = 'server.key'
|
|
|
308
326
|
ssl_ca_file = 'ca.crt'
|
|
309
327
|
```
|
|
310
328
|
|
|
329
|
+
### Auditing Live Server Settings
|
|
330
|
+
|
|
331
|
+
Add `--audit` to connect to the live server and compare current settings against recommendations:
|
|
332
|
+
|
|
333
|
+
```bash
|
|
334
|
+
pgmonkey pgserverconfig --filepath /path/to/config.yaml --audit
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
This queries the server's `pg_settings` (read-only) and displays a comparison table:
|
|
338
|
+
|
|
339
|
+
```
|
|
340
|
+
1) Database type detected: PostgreSQL
|
|
341
|
+
|
|
342
|
+
2) Server settings audit:
|
|
343
|
+
|
|
344
|
+
postgresql.conf:
|
|
345
|
+
|
|
346
|
+
Setting Recommended Current Source Status
|
|
347
|
+
────────────────────────────────────────────────────────────────────
|
|
348
|
+
max_connections 22 100 configuration file OK
|
|
349
|
+
ssl on on configuration file OK
|
|
350
|
+
ssl_cert_file server.crt server.crt configuration file OK
|
|
351
|
+
ssl_key_file server.key server.key configuration file OK
|
|
352
|
+
ssl_ca_file ca.crt ca.crt configuration file OK
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
356
|
+
|
|
357
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified — the audit is entirely read-only.
|
|
358
|
+
|
|
311
359
|
### Importing and Exporting Data
|
|
312
360
|
|
|
313
361
|
**Import data** from a CSV or text file into a PostgreSQL table:
|
|
@@ -461,7 +509,8 @@ pgmonkey handles several production concerns behind the scenes so you don't have
|
|
|
461
509
|
- **Connection caching** — Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
462
510
|
- **Async pool lifecycle** — `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
463
511
|
- **atexit cleanup** — All cached connections are automatically closed when the process exits.
|
|
464
|
-
- **Thread-safe caching** — The connection cache is protected by a threading lock.
|
|
512
|
+
- **Thread-safe caching** — The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
513
|
+
- **Config validation** — Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
465
514
|
|
|
466
515
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
467
516
|
|
|
@@ -693,7 +742,7 @@ Run the tests:
|
|
|
693
742
|
pytest
|
|
694
743
|
```
|
|
695
744
|
|
|
696
|
-
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation
|
|
745
|
+
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, server config generation, and server settings inspection.
|
|
697
746
|
|
|
698
747
|
---
|
|
699
748
|
|
|
@@ -9,6 +9,9 @@ def cli_pg_server_config_subparser(subparsers):
|
|
|
9
9
|
|
|
10
10
|
pg_server_config_parser.add_argument('--filepath', required=True,
|
|
11
11
|
help='Path to the config you want settings generated.')
|
|
12
|
+
pg_server_config_parser.add_argument('--audit', action='store_true', default=False,
|
|
13
|
+
help='Connect to the server and compare current settings '
|
|
14
|
+
'against recommendations.')
|
|
12
15
|
pg_server_config_parser.set_defaults(func=pg_server_config_create_handler,
|
|
13
16
|
pg_server_config_manager=pg_server_config_manager)
|
|
14
17
|
|
|
@@ -16,5 +19,7 @@ def cli_pg_server_config_subparser(subparsers):
|
|
|
16
19
|
def pg_server_config_create_handler(args):
|
|
17
20
|
pg_server_config_manager = args.pg_server_config_manager
|
|
18
21
|
|
|
19
|
-
if args.
|
|
22
|
+
if args.audit:
|
|
23
|
+
pg_server_config_manager.audit_server_config(args.filepath)
|
|
24
|
+
else:
|
|
20
25
|
pg_server_config_manager.get_server_config(args.filepath)
|
|
@@ -36,6 +36,10 @@ def cli_pgconfig_subparser(subparsers):
|
|
|
36
36
|
help='Path to the configuration file you want to use.')
|
|
37
37
|
code_parser.add_argument('--connection-type', choices=CONNECTION_TYPE_CHOICES, default=None,
|
|
38
38
|
help='Connection type for generated code. Overrides the value in the config file.')
|
|
39
|
+
code_parser.add_argument('--library', choices=['pgmonkey', 'psycopg'], default='pgmonkey',
|
|
40
|
+
help='Target library for generated code. '
|
|
41
|
+
'"pgmonkey" (default) generates code using pgmonkey. '
|
|
42
|
+
'"psycopg" generates code using psycopg/psycopg_pool directly.')
|
|
39
43
|
code_parser.set_defaults(func=pgconfig_generate_code_handler, pgcodegen_manager=pgcodegen_manager)
|
|
40
44
|
|
|
41
45
|
|
|
@@ -58,4 +62,5 @@ def pgconfig_test_handler(args):
|
|
|
58
62
|
def pgconfig_generate_code_handler(args):
|
|
59
63
|
pgcodegen_manager = args.pgcodegen_manager
|
|
60
64
|
connection_type = getattr(args, 'connection_type', None)
|
|
61
|
-
|
|
65
|
+
library = getattr(args, 'library', 'pgmonkey')
|
|
66
|
+
pgcodegen_manager.generate_connection_code(args.filepath, connection_type, library=library)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
postgresql:
|
|
2
|
+
# Default connection type used when none is specified in the API call.
|
|
3
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
4
|
+
# You can override this per-call:
|
|
5
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
6
|
+
connection_type: 'normal'
|
|
7
|
+
|
|
8
|
+
connection_settings:
|
|
9
|
+
user: 'postgres'
|
|
10
|
+
password: 'password'
|
|
11
|
+
host: 'localhost'
|
|
12
|
+
port: '5432'
|
|
13
|
+
dbname: 'mydatabase'
|
|
14
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
15
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
16
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
17
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
18
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
19
|
+
application_name: 'myapp'
|
|
20
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
21
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
22
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
23
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
24
|
+
# autocommit: false # Enable autocommit mode (required for DDL like CREATE DATABASE, VACUUM, etc.)
|
|
25
|
+
|
|
26
|
+
# Session-level GUC settings for 'normal' and 'pool' connection types.
|
|
27
|
+
# For 'normal': applied via SET commands after connect.
|
|
28
|
+
# For 'pool': applied via psycopg_pool's configure callback on each checkout.
|
|
29
|
+
sync_settings:
|
|
30
|
+
# statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
31
|
+
# lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
32
|
+
# idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
33
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
34
|
+
|
|
35
|
+
# Settings for 'pool' connection type
|
|
36
|
+
pool_settings:
|
|
37
|
+
min_size: 5
|
|
38
|
+
max_size: 20
|
|
39
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
40
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
41
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
42
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
43
|
+
# max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
|
|
44
|
+
# reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
|
|
45
|
+
# num_workers: 3 # Number of background workers maintaining pool connections
|
|
46
|
+
|
|
47
|
+
# Session-level GUC settings for 'async' and 'async_pool' connection types.
|
|
48
|
+
# For 'async': applied via SET commands after connect.
|
|
49
|
+
# For 'async_pool': applied via psycopg_pool's configure callback on each checkout.
|
|
50
|
+
async_settings:
|
|
51
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
52
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
53
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
54
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
55
|
+
|
|
56
|
+
# Settings for 'async_pool' connection type
|
|
57
|
+
async_pool_settings:
|
|
58
|
+
min_size: 5
|
|
59
|
+
max_size: 20
|
|
60
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
61
|
+
max_idle: 300
|
|
62
|
+
max_lifetime: 3600
|
|
63
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
64
|
+
# max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
|
|
65
|
+
# reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
|
|
66
|
+
# num_workers: 3 # Number of background workers maintaining pool connections
|
|
@@ -1,28 +1,34 @@
|
|
|
1
|
-
|
|
1
|
+
import logging
|
|
2
|
+
from psycopg import AsyncConnection, OperationalError, sql
|
|
2
3
|
from .base_connection import PostgresBaseConnection
|
|
3
4
|
from contextlib import asynccontextmanager
|
|
4
5
|
from typing import Optional
|
|
5
6
|
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
6
9
|
|
|
7
10
|
class PGAsyncConnection(PostgresBaseConnection):
|
|
8
11
|
def __init__(self, config, async_settings=None):
|
|
9
12
|
self.config = config
|
|
10
13
|
self.async_settings = async_settings or {}
|
|
14
|
+
self.autocommit = None
|
|
11
15
|
self.connection: Optional[AsyncConnection] = None
|
|
12
16
|
|
|
13
17
|
async def connect(self):
|
|
14
18
|
"""Establishes an asynchronous database connection."""
|
|
15
19
|
if self.connection is None or self.connection.closed:
|
|
16
|
-
self.connection = await AsyncConnection.connect(
|
|
20
|
+
self.connection = await AsyncConnection.connect(
|
|
21
|
+
autocommit=bool(self.autocommit), **self.config
|
|
22
|
+
)
|
|
17
23
|
await self._apply_async_settings()
|
|
18
24
|
|
|
19
25
|
async def _apply_async_settings(self):
|
|
20
26
|
"""Applies PostgreSQL GUC settings via SET commands after connection is established."""
|
|
21
27
|
for setting, value in self.async_settings.items():
|
|
22
28
|
try:
|
|
23
|
-
await self.connection.execute(
|
|
29
|
+
await self.connection.execute(sql.SQL("SET {} = %s").format(sql.Identifier(setting)), (str(value),))
|
|
24
30
|
except Exception as e:
|
|
25
|
-
|
|
31
|
+
logger.warning("Could not apply setting '%s': %s", setting, e)
|
|
26
32
|
|
|
27
33
|
async def test_connection(self):
|
|
28
34
|
"""Tests the asynchronous database connection."""
|
|
@@ -30,11 +36,11 @@ class PGAsyncConnection(PostgresBaseConnection):
|
|
|
30
36
|
async with self.cursor() as cur:
|
|
31
37
|
await cur.execute('SELECT 1;')
|
|
32
38
|
result = await cur.fetchone()
|
|
33
|
-
|
|
39
|
+
logger.info("Async connection successful: %s", result)
|
|
34
40
|
except OperationalError as e:
|
|
35
|
-
|
|
41
|
+
logger.error("Connection failed: %s", e)
|
|
36
42
|
except Exception as e:
|
|
37
|
-
|
|
43
|
+
logger.error("An unexpected error occurred: %s", e)
|
|
38
44
|
|
|
39
45
|
async def disconnect(self):
|
|
40
46
|
"""Closes the asynchronous database connection."""
|
|
@@ -55,7 +61,7 @@ class PGAsyncConnection(PostgresBaseConnection):
|
|
|
55
61
|
"""Creates a transaction context on the async connection."""
|
|
56
62
|
if self.connection:
|
|
57
63
|
async with self.connection.transaction():
|
|
58
|
-
yield
|
|
64
|
+
yield self
|
|
59
65
|
else:
|
|
60
66
|
raise Exception("No active connection available for transaction")
|
|
61
67
|
|
|
@@ -73,8 +79,10 @@ class PGAsyncConnection(PostgresBaseConnection):
|
|
|
73
79
|
return self
|
|
74
80
|
|
|
75
81
|
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
82
|
+
try:
|
|
83
|
+
if exc_type:
|
|
84
|
+
await self.rollback()
|
|
85
|
+
else:
|
|
86
|
+
await self.commit()
|
|
87
|
+
finally:
|
|
88
|
+
await self.disconnect()
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import warnings
|
|
3
|
+
import contextvars
|
|
4
|
+
from psycopg_pool import AsyncConnectionPool
|
|
5
|
+
from psycopg import conninfo as psycopg_conninfo, sql
|
|
6
|
+
from .base_connection import PostgresBaseConnection
|
|
7
|
+
from contextlib import asynccontextmanager
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
warnings.filterwarnings('ignore', category=RuntimeWarning, module='psycopg_pool')
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PGAsyncPoolConnection(PostgresBaseConnection):
|
|
15
|
+
def __init__(self, config, async_pool_settings=None, async_settings=None):
|
|
16
|
+
self.config = config
|
|
17
|
+
self.async_pool_settings = async_pool_settings or {}
|
|
18
|
+
self.async_settings = async_settings or {}
|
|
19
|
+
self.pool = None
|
|
20
|
+
self._pool_conn = contextvars.ContextVar(f'_pool_conn_{id(self)}', default=None)
|
|
21
|
+
self._pool_conn_ctx = contextvars.ContextVar(f'_pool_ctx_{id(self)}', default=None)
|
|
22
|
+
|
|
23
|
+
@staticmethod
|
|
24
|
+
def construct_conninfo(config):
|
|
25
|
+
"""Constructs a properly escaped connection info string from the config dictionary."""
|
|
26
|
+
return psycopg_conninfo.make_conninfo(**config)
|
|
27
|
+
|
|
28
|
+
async def connect(self):
|
|
29
|
+
"""Initialize the async connection pool."""
|
|
30
|
+
if self.pool is None:
|
|
31
|
+
conninfo = self.construct_conninfo(self.config)
|
|
32
|
+
kwargs = dict(self.async_pool_settings)
|
|
33
|
+
|
|
34
|
+
check_on_checkout = kwargs.pop('check_on_checkout', False)
|
|
35
|
+
if check_on_checkout:
|
|
36
|
+
async def _check(conn):
|
|
37
|
+
await conn.execute("SELECT 1")
|
|
38
|
+
kwargs['check'] = _check
|
|
39
|
+
|
|
40
|
+
if self.async_settings:
|
|
41
|
+
async_settings = self.async_settings
|
|
42
|
+
async def _configure(conn):
|
|
43
|
+
for setting, value in async_settings.items():
|
|
44
|
+
try:
|
|
45
|
+
await conn.execute(sql.SQL("SET {} = %s").format(sql.Identifier(setting)), (str(value),))
|
|
46
|
+
except Exception as e:
|
|
47
|
+
logger.warning("Could not apply setting '%s': %s", setting, e)
|
|
48
|
+
kwargs['configure'] = _configure
|
|
49
|
+
|
|
50
|
+
self.pool = AsyncConnectionPool(conninfo=conninfo, **kwargs)
|
|
51
|
+
await self.pool.open()
|
|
52
|
+
|
|
53
|
+
async def test_connection(self):
|
|
54
|
+
"""Tests a single connection from the async pool."""
|
|
55
|
+
if not self.pool:
|
|
56
|
+
await self.connect()
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
async with self.pool.connection() as conn:
|
|
60
|
+
async with conn.cursor() as cur:
|
|
61
|
+
await cur.execute('SELECT 1;')
|
|
62
|
+
result = await cur.fetchone()
|
|
63
|
+
logger.info("Async pool connection successful: %s", result)
|
|
64
|
+
except Exception as e:
|
|
65
|
+
logger.error("Test connection failed: %s", e)
|
|
66
|
+
|
|
67
|
+
async def disconnect(self):
|
|
68
|
+
if self.pool:
|
|
69
|
+
await self.pool.close()
|
|
70
|
+
self.pool = None
|
|
71
|
+
|
|
72
|
+
async def commit(self):
|
|
73
|
+
"""Commits the current transaction on the acquired connection."""
|
|
74
|
+
conn = self._pool_conn.get()
|
|
75
|
+
if conn:
|
|
76
|
+
await conn.commit()
|
|
77
|
+
|
|
78
|
+
async def rollback(self):
|
|
79
|
+
"""Rolls back the current transaction on the acquired connection."""
|
|
80
|
+
conn = self._pool_conn.get()
|
|
81
|
+
if conn:
|
|
82
|
+
await conn.rollback()
|
|
83
|
+
|
|
84
|
+
@asynccontextmanager
|
|
85
|
+
async def transaction(self):
|
|
86
|
+
"""Creates a transaction context on a pooled connection."""
|
|
87
|
+
conn = self._pool_conn.get()
|
|
88
|
+
if conn:
|
|
89
|
+
# Inside __aenter__/__aexit__ context - use the acquired connection
|
|
90
|
+
async with conn.transaction():
|
|
91
|
+
yield self
|
|
92
|
+
elif self.pool:
|
|
93
|
+
# Standalone usage - acquire connection from pool
|
|
94
|
+
async with self.pool.connection() as acquired:
|
|
95
|
+
token = self._pool_conn.set(acquired)
|
|
96
|
+
try:
|
|
97
|
+
async with acquired.transaction():
|
|
98
|
+
yield self
|
|
99
|
+
finally:
|
|
100
|
+
self._pool_conn.reset(token)
|
|
101
|
+
else:
|
|
102
|
+
raise Exception("No active pool available for transaction")
|
|
103
|
+
|
|
104
|
+
@asynccontextmanager
|
|
105
|
+
async def cursor(self):
|
|
106
|
+
"""Provides an async cursor from a pooled connection."""
|
|
107
|
+
conn = self._pool_conn.get()
|
|
108
|
+
if conn:
|
|
109
|
+
# Inside __aenter__/__aexit__ context - use the acquired connection
|
|
110
|
+
async with conn.cursor() as cur:
|
|
111
|
+
yield cur
|
|
112
|
+
elif self.pool:
|
|
113
|
+
# Standalone usage - acquire connection from pool
|
|
114
|
+
async with self.pool.connection() as acquired:
|
|
115
|
+
async with acquired.cursor() as cur:
|
|
116
|
+
yield cur
|
|
117
|
+
else:
|
|
118
|
+
raise Exception("No active pool available for cursor")
|
|
119
|
+
|
|
120
|
+
async def __aenter__(self):
|
|
121
|
+
if not self.pool:
|
|
122
|
+
await self.connect()
|
|
123
|
+
pool_conn_ctx = self.pool.connection()
|
|
124
|
+
conn = await pool_conn_ctx.__aenter__()
|
|
125
|
+
self._pool_conn.set(conn)
|
|
126
|
+
self._pool_conn_ctx.set(pool_conn_ctx)
|
|
127
|
+
return self
|
|
128
|
+
|
|
129
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
130
|
+
try:
|
|
131
|
+
conn = self._pool_conn.get()
|
|
132
|
+
if exc_type:
|
|
133
|
+
if conn:
|
|
134
|
+
await conn.rollback()
|
|
135
|
+
else:
|
|
136
|
+
if conn:
|
|
137
|
+
await conn.commit()
|
|
138
|
+
finally:
|
|
139
|
+
pool_conn_ctx = self._pool_conn_ctx.get()
|
|
140
|
+
if pool_conn_ctx:
|
|
141
|
+
await pool_conn_ctx.__aexit__(exc_type, exc_val, exc_tb)
|
|
142
|
+
self._pool_conn.set(None)
|
|
143
|
+
self._pool_conn_ctx.set(None)
|