pgmonkey 2.2.0__tar.gz → 3.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgmonkey-2.2.0/src/pgmonkey.egg-info → pgmonkey-3.2.0}/PKG-INFO +114 -87
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/README.md +112 -85
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/pyproject.toml +2 -2
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_export_subparser.py +6 -2
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_import_subparser.py +6 -2
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_pg_server_config_subparser.py +6 -1
- pgmonkey-3.2.0/src/pgmonkey/common/exceptions.py +3 -0
- pgmonkey-3.2.0/src/pgmonkey/common/templates/postgres.yaml +65 -0
- pgmonkey-3.2.0/src/pgmonkey/common/utils/configutils.py +26 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/async_connection.py +14 -9
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/async_pool_connection.py +47 -31
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/normal_connection.py +24 -9
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/pool_connection.py +61 -26
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/postgres_connection_factory.py +13 -8
- pgmonkey-3.2.0/src/pgmonkey/managers/pg_server_config_manager.py +47 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/pgcodegen_manager.py +6 -8
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/pgconfig_manager.py +4 -7
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/pgconnection_manager.py +6 -3
- pgmonkey-3.2.0/src/pgmonkey/serversettings/postgres_server_config_generator.py +209 -0
- pgmonkey-3.2.0/src/pgmonkey/serversettings/postgres_server_settings_inspector.py +155 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/conftest.py +48 -42
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_async_connection.py +77 -3
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_async_pool_connection.py +292 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_code_generator.py +40 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_config_manager.py +5 -5
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_connection_caching.py +43 -24
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_connection_factory.py +77 -13
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_csv_data_exporter.py +132 -0
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_csv_data_importer.py +193 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_normal_connection.py +101 -3
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_pgconnection_manager.py +1 -1
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_pool_connection.py +275 -0
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_server_config_generator.py +295 -0
- pgmonkey-3.2.0/src/pgmonkey/tests/unit/test_server_settings_inspector.py +243 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tools/connection_code_generator.py +27 -25
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tools/csv_data_exporter.py +34 -18
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tools/csv_data_importer.py +103 -72
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tools/database_connection_tester.py +3 -1
- {pgmonkey-2.2.0 → pgmonkey-3.2.0/src/pgmonkey.egg-info}/PKG-INFO +114 -87
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey.egg-info/SOURCES.txt +6 -0
- pgmonkey-2.2.0/src/pgmonkey/common/templates/postgres.yaml +0 -48
- pgmonkey-2.2.0/src/pgmonkey/managers/pg_server_config_manager.py +0 -22
- pgmonkey-2.2.0/src/pgmonkey/serversettings/postgres_server_config_generator.py +0 -118
- pgmonkey-2.2.0/src/pgmonkey/tests/unit/test_async_pool_connection.py +0 -87
- pgmonkey-2.2.0/src/pgmonkey/tests/unit/test_pool_connection.py +0 -123
- pgmonkey-2.2.0/src/pgmonkey/tests/unit/test_server_config_generator.py +0 -152
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/LICENSE +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/NOTICE +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/setup.cfg +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_pgconfig_subparser.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_settings_subparser.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/cli/cli_toplevel_parser.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/common/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/common/config/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/common/utils/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/common/utils/pathutils.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/base.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/connections/postgres/base_connection.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/pgexport_manager.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/pgimport_manager.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/settings_manager.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/managers/toplevel_manager.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/settings/app_settings.yaml +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/integration/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/integration/test_pgconnection_manager_integration.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_base_connection.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_path_utils.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tests/unit/test_settings_manager.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey/tools/__init__.py +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey.egg-info/dependency_links.txt +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey.egg-info/entry_points.txt +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey.egg-info/requires.txt +0 -0
- {pgmonkey-2.2.0 → pgmonkey-3.2.0}/src/pgmonkey.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgmonkey
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.2.0
|
|
4
4
|
Summary: A tool to assist with postgresql database connections
|
|
5
5
|
Author-email: Good Boy <pythonic@rexbytes.com>
|
|
6
6
|
License: MIT
|
|
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Topic :: Database
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
-
Requires-Python: <
|
|
18
|
+
Requires-Python: <4.0,>=3.10
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: NOTICE
|
|
@@ -52,6 +52,7 @@ Dynamic: license-file
|
|
|
52
52
|
- [Testing a Connection](#testing-a-connection)
|
|
53
53
|
- [Generating Python Code](#generating-python-code)
|
|
54
54
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
55
|
+
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
55
56
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
56
57
|
6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
57
58
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
@@ -89,7 +90,7 @@ pip install pgmonkey[test]
|
|
|
89
90
|
|
|
90
91
|
## One Config File, All Connection Types
|
|
91
92
|
|
|
92
|
-
|
|
93
|
+
pgmonkey uses a **single YAML configuration file** for all connection types. Instead of maintaining separate config files for normal, pool, async, and async_pool connections, you define everything in one file and specify the connection type when you call the API:
|
|
93
94
|
|
|
94
95
|
```python
|
|
95
96
|
from pgmonkey import PGConnectionManager
|
|
@@ -110,55 +111,54 @@ The `connection_type` parameter is optional. If omitted, pgmonkey uses the `conn
|
|
|
110
111
|
Here is the full configuration template. You only need to fill in the sections relevant to the connection types you plan to use.
|
|
111
112
|
|
|
112
113
|
```yaml
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
114
|
+
# Default connection type when none is specified in the API call.
|
|
115
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
116
|
+
# You can override this per-call:
|
|
117
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
118
|
+
connection_type: 'normal'
|
|
119
|
+
|
|
120
|
+
connection_settings:
|
|
121
|
+
user: 'postgres'
|
|
122
|
+
password: 'password'
|
|
123
|
+
host: 'localhost'
|
|
124
|
+
port: '5432'
|
|
125
|
+
dbname: 'mydatabase'
|
|
126
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
127
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
128
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
129
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
130
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
131
|
+
application_name: 'myapp'
|
|
132
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
133
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
134
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
135
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
136
|
+
|
|
137
|
+
# Settings for 'pool' connection type
|
|
138
|
+
pool_settings:
|
|
139
|
+
min_size: 5
|
|
140
|
+
max_size: 20
|
|
141
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
142
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
143
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
144
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
145
|
+
|
|
146
|
+
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
147
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
148
|
+
async_settings:
|
|
149
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
150
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
151
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
152
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
153
|
+
|
|
154
|
+
# Settings for 'async_pool' connection type
|
|
155
|
+
async_pool_settings:
|
|
156
|
+
min_size: 5
|
|
157
|
+
max_size: 20
|
|
158
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
159
|
+
max_idle: 300
|
|
160
|
+
max_lifetime: 3600
|
|
161
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
162
162
|
```
|
|
163
163
|
|
|
164
164
|
### Connection Settings
|
|
@@ -225,13 +225,12 @@ Used by `async_pool` connection type. Same parameters as pool settings. The `asy
|
|
|
225
225
|
The most common method. Credentials are sent to the PostgreSQL server for validation.
|
|
226
226
|
|
|
227
227
|
```yaml
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
dbname: 'your_database'
|
|
228
|
+
connection_type: 'normal'
|
|
229
|
+
connection_settings:
|
|
230
|
+
user: 'your_user'
|
|
231
|
+
password: 'your_password'
|
|
232
|
+
host: 'localhost'
|
|
233
|
+
dbname: 'your_database'
|
|
235
234
|
```
|
|
236
235
|
|
|
237
236
|
### SSL/TLS Encryption
|
|
@@ -246,15 +245,14 @@ SSL/TLS encrypts the connection between your application and the PostgreSQL serv
|
|
|
246
245
|
- `verify-full`: Require SSL, verify certificate, and ensure the hostname matches.
|
|
247
246
|
|
|
248
247
|
```yaml
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
sslrootcert: '/path/to/ca.crt'
|
|
248
|
+
connection_type: 'normal'
|
|
249
|
+
connection_settings:
|
|
250
|
+
user: 'your_user'
|
|
251
|
+
password: 'your_password'
|
|
252
|
+
host: 'localhost'
|
|
253
|
+
dbname: 'your_database'
|
|
254
|
+
sslmode: 'require'
|
|
255
|
+
sslrootcert: '/path/to/ca.crt'
|
|
258
256
|
```
|
|
259
257
|
|
|
260
258
|
### Certificate-Based Authentication
|
|
@@ -262,17 +260,16 @@ postgresql:
|
|
|
262
260
|
Uses SSL client certificates for authentication. Highly secure and often used in enterprise environments.
|
|
263
261
|
|
|
264
262
|
```yaml
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
sslrootcert: '/path/to/ca.crt'
|
|
263
|
+
connection_type: 'normal'
|
|
264
|
+
connection_settings:
|
|
265
|
+
user: 'your_user'
|
|
266
|
+
password: 'your_password'
|
|
267
|
+
host: 'localhost'
|
|
268
|
+
dbname: 'your_database'
|
|
269
|
+
sslmode: 'verify-full'
|
|
270
|
+
sslcert: '/path/to/client.crt'
|
|
271
|
+
sslkey: '/path/to/client.key'
|
|
272
|
+
sslrootcert: '/path/to/ca.crt'
|
|
276
273
|
```
|
|
277
274
|
|
|
278
275
|
## Using the CLI
|
|
@@ -325,8 +322,8 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-typ
|
|
|
325
322
|
|
|
326
323
|
The `--library` flag controls which library the generated code targets:
|
|
327
324
|
|
|
328
|
-
- `pgmonkey` (default)
|
|
329
|
-
- `psycopg`
|
|
325
|
+
- `pgmonkey` (default) - generates code using pgmonkey's `PGConnectionManager`.
|
|
326
|
+
- `psycopg` - generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
330
327
|
|
|
331
328
|
### Server Configuration Recommendations
|
|
332
329
|
|
|
@@ -357,6 +354,36 @@ ssl_key_file = 'server.key'
|
|
|
357
354
|
ssl_ca_file = 'ca.crt'
|
|
358
355
|
```
|
|
359
356
|
|
|
357
|
+
### Auditing Live Server Settings
|
|
358
|
+
|
|
359
|
+
Add `--audit` to connect to the live server and compare current settings against recommendations:
|
|
360
|
+
|
|
361
|
+
```bash
|
|
362
|
+
pgmonkey pgserverconfig --filepath /path/to/config.yaml --audit
|
|
363
|
+
```
|
|
364
|
+
|
|
365
|
+
This queries the server's `pg_settings` (read-only) and displays a comparison table:
|
|
366
|
+
|
|
367
|
+
```
|
|
368
|
+
1) Database type detected: PostgreSQL
|
|
369
|
+
|
|
370
|
+
2) Server settings audit:
|
|
371
|
+
|
|
372
|
+
postgresql.conf:
|
|
373
|
+
|
|
374
|
+
Setting Recommended Current Source Status
|
|
375
|
+
────────────────────────────────────────────────────────────────────
|
|
376
|
+
max_connections 22 100 configuration file OK
|
|
377
|
+
ssl on on configuration file OK
|
|
378
|
+
ssl_cert_file server.crt server.crt configuration file OK
|
|
379
|
+
ssl_key_file server.key server.key configuration file OK
|
|
380
|
+
ssl_ca_file ca.crt ca.crt configuration file OK
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
384
|
+
|
|
385
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified - the audit is entirely read-only.
|
|
386
|
+
|
|
360
387
|
### Importing and Exporting Data
|
|
361
388
|
|
|
362
389
|
**Import data** from a CSV or text file into a PostgreSQL table:
|
|
@@ -507,11 +534,11 @@ except Exception as e:
|
|
|
507
534
|
|
|
508
535
|
pgmonkey handles several production concerns behind the scenes so you don't have to:
|
|
509
536
|
|
|
510
|
-
- **Connection caching**
|
|
511
|
-
- **Async pool lifecycle**
|
|
512
|
-
- **atexit cleanup**
|
|
513
|
-
- **Thread-safe caching**
|
|
514
|
-
- **Config validation**
|
|
537
|
+
- **Connection caching** - Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
538
|
+
- **Async pool lifecycle** - `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
539
|
+
- **atexit cleanup** - All cached connections are automatically closed when the process exits.
|
|
540
|
+
- **Thread-safe caching** - The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
541
|
+
- **Config validation** - Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
515
542
|
|
|
516
543
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
517
544
|
|
|
@@ -743,7 +770,7 @@ Run the tests:
|
|
|
743
770
|
pytest
|
|
744
771
|
```
|
|
745
772
|
|
|
746
|
-
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, and server
|
|
773
|
+
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, server config generation, and server settings inspection.
|
|
747
774
|
|
|
748
775
|
---
|
|
749
776
|
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
- [Testing a Connection](#testing-a-connection)
|
|
21
21
|
- [Generating Python Code](#generating-python-code)
|
|
22
22
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
23
|
+
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
23
24
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
24
25
|
6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
25
26
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
@@ -57,7 +58,7 @@ pip install pgmonkey[test]
|
|
|
57
58
|
|
|
58
59
|
## One Config File, All Connection Types
|
|
59
60
|
|
|
60
|
-
|
|
61
|
+
pgmonkey uses a **single YAML configuration file** for all connection types. Instead of maintaining separate config files for normal, pool, async, and async_pool connections, you define everything in one file and specify the connection type when you call the API:
|
|
61
62
|
|
|
62
63
|
```python
|
|
63
64
|
from pgmonkey import PGConnectionManager
|
|
@@ -78,55 +79,54 @@ The `connection_type` parameter is optional. If omitted, pgmonkey uses the `conn
|
|
|
78
79
|
Here is the full configuration template. You only need to fill in the sections relevant to the connection types you plan to use.
|
|
79
80
|
|
|
80
81
|
```yaml
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
82
|
+
# Default connection type when none is specified in the API call.
|
|
83
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
84
|
+
# You can override this per-call:
|
|
85
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
86
|
+
connection_type: 'normal'
|
|
87
|
+
|
|
88
|
+
connection_settings:
|
|
89
|
+
user: 'postgres'
|
|
90
|
+
password: 'password'
|
|
91
|
+
host: 'localhost'
|
|
92
|
+
port: '5432'
|
|
93
|
+
dbname: 'mydatabase'
|
|
94
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
95
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
96
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
97
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
98
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
99
|
+
application_name: 'myapp'
|
|
100
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
101
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
102
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
103
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
104
|
+
|
|
105
|
+
# Settings for 'pool' connection type
|
|
106
|
+
pool_settings:
|
|
107
|
+
min_size: 5
|
|
108
|
+
max_size: 20
|
|
109
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
110
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
111
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
112
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
113
|
+
|
|
114
|
+
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
115
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
116
|
+
async_settings:
|
|
117
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
118
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
119
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
120
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
121
|
+
|
|
122
|
+
# Settings for 'async_pool' connection type
|
|
123
|
+
async_pool_settings:
|
|
124
|
+
min_size: 5
|
|
125
|
+
max_size: 20
|
|
126
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
127
|
+
max_idle: 300
|
|
128
|
+
max_lifetime: 3600
|
|
129
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
130
130
|
```
|
|
131
131
|
|
|
132
132
|
### Connection Settings
|
|
@@ -193,13 +193,12 @@ Used by `async_pool` connection type. Same parameters as pool settings. The `asy
|
|
|
193
193
|
The most common method. Credentials are sent to the PostgreSQL server for validation.
|
|
194
194
|
|
|
195
195
|
```yaml
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
dbname: 'your_database'
|
|
196
|
+
connection_type: 'normal'
|
|
197
|
+
connection_settings:
|
|
198
|
+
user: 'your_user'
|
|
199
|
+
password: 'your_password'
|
|
200
|
+
host: 'localhost'
|
|
201
|
+
dbname: 'your_database'
|
|
203
202
|
```
|
|
204
203
|
|
|
205
204
|
### SSL/TLS Encryption
|
|
@@ -214,15 +213,14 @@ SSL/TLS encrypts the connection between your application and the PostgreSQL serv
|
|
|
214
213
|
- `verify-full`: Require SSL, verify certificate, and ensure the hostname matches.
|
|
215
214
|
|
|
216
215
|
```yaml
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
sslrootcert: '/path/to/ca.crt'
|
|
216
|
+
connection_type: 'normal'
|
|
217
|
+
connection_settings:
|
|
218
|
+
user: 'your_user'
|
|
219
|
+
password: 'your_password'
|
|
220
|
+
host: 'localhost'
|
|
221
|
+
dbname: 'your_database'
|
|
222
|
+
sslmode: 'require'
|
|
223
|
+
sslrootcert: '/path/to/ca.crt'
|
|
226
224
|
```
|
|
227
225
|
|
|
228
226
|
### Certificate-Based Authentication
|
|
@@ -230,17 +228,16 @@ postgresql:
|
|
|
230
228
|
Uses SSL client certificates for authentication. Highly secure and often used in enterprise environments.
|
|
231
229
|
|
|
232
230
|
```yaml
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
sslrootcert: '/path/to/ca.crt'
|
|
231
|
+
connection_type: 'normal'
|
|
232
|
+
connection_settings:
|
|
233
|
+
user: 'your_user'
|
|
234
|
+
password: 'your_password'
|
|
235
|
+
host: 'localhost'
|
|
236
|
+
dbname: 'your_database'
|
|
237
|
+
sslmode: 'verify-full'
|
|
238
|
+
sslcert: '/path/to/client.crt'
|
|
239
|
+
sslkey: '/path/to/client.key'
|
|
240
|
+
sslrootcert: '/path/to/ca.crt'
|
|
244
241
|
```
|
|
245
242
|
|
|
246
243
|
## Using the CLI
|
|
@@ -293,8 +290,8 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-typ
|
|
|
293
290
|
|
|
294
291
|
The `--library` flag controls which library the generated code targets:
|
|
295
292
|
|
|
296
|
-
- `pgmonkey` (default)
|
|
297
|
-
- `psycopg`
|
|
293
|
+
- `pgmonkey` (default) - generates code using pgmonkey's `PGConnectionManager`.
|
|
294
|
+
- `psycopg` - generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
298
295
|
|
|
299
296
|
### Server Configuration Recommendations
|
|
300
297
|
|
|
@@ -325,6 +322,36 @@ ssl_key_file = 'server.key'
|
|
|
325
322
|
ssl_ca_file = 'ca.crt'
|
|
326
323
|
```
|
|
327
324
|
|
|
325
|
+
### Auditing Live Server Settings
|
|
326
|
+
|
|
327
|
+
Add `--audit` to connect to the live server and compare current settings against recommendations:
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
pgmonkey pgserverconfig --filepath /path/to/config.yaml --audit
|
|
331
|
+
```
|
|
332
|
+
|
|
333
|
+
This queries the server's `pg_settings` (read-only) and displays a comparison table:
|
|
334
|
+
|
|
335
|
+
```
|
|
336
|
+
1) Database type detected: PostgreSQL
|
|
337
|
+
|
|
338
|
+
2) Server settings audit:
|
|
339
|
+
|
|
340
|
+
postgresql.conf:
|
|
341
|
+
|
|
342
|
+
Setting Recommended Current Source Status
|
|
343
|
+
────────────────────────────────────────────────────────────────────
|
|
344
|
+
max_connections 22 100 configuration file OK
|
|
345
|
+
ssl on on configuration file OK
|
|
346
|
+
ssl_cert_file server.crt server.crt configuration file OK
|
|
347
|
+
ssl_key_file server.key server.key configuration file OK
|
|
348
|
+
ssl_ca_file ca.crt ca.crt configuration file OK
|
|
349
|
+
```
|
|
350
|
+
|
|
351
|
+
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
352
|
+
|
|
353
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified - the audit is entirely read-only.
|
|
354
|
+
|
|
328
355
|
### Importing and Exporting Data
|
|
329
356
|
|
|
330
357
|
**Import data** from a CSV or text file into a PostgreSQL table:
|
|
@@ -475,11 +502,11 @@ except Exception as e:
|
|
|
475
502
|
|
|
476
503
|
pgmonkey handles several production concerns behind the scenes so you don't have to:
|
|
477
504
|
|
|
478
|
-
- **Connection caching**
|
|
479
|
-
- **Async pool lifecycle**
|
|
480
|
-
- **atexit cleanup**
|
|
481
|
-
- **Thread-safe caching**
|
|
482
|
-
- **Config validation**
|
|
505
|
+
- **Connection caching** - Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
506
|
+
- **Async pool lifecycle** - `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
507
|
+
- **atexit cleanup** - All cached connections are automatically closed when the process exits.
|
|
508
|
+
- **Thread-safe caching** - The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
509
|
+
- **Config validation** - Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
483
510
|
|
|
484
511
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
485
512
|
|
|
@@ -711,7 +738,7 @@ Run the tests:
|
|
|
711
738
|
pytest
|
|
712
739
|
```
|
|
713
740
|
|
|
714
|
-
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, and server
|
|
741
|
+
The test suite uses mocks and covers all connection types, the connection factory, configuration management, code generation (both pgmonkey and native psycopg), config validation, server config generation, and server settings inspection.
|
|
715
742
|
|
|
716
743
|
---
|
|
717
744
|
|
|
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pgmonkey"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.2.0"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Good Boy", email="pythonic@rexbytes.com" },
|
|
10
10
|
]
|
|
11
11
|
description = "A tool to assist with postgresql database connections"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = {text = "MIT"}
|
|
14
|
-
requires-python = ">=3.10,<
|
|
14
|
+
requires-python = ">=3.10,<4.0"
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Programming Language :: Python :: 3.10",
|
|
17
17
|
"Programming Language :: Python :: 3.11",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pgmonkey.managers.pgexport_manager import PGExportManager
|
|
2
|
+
from pgmonkey.common.exceptions import ConfigFileCreatedError
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
|
|
@@ -41,5 +42,8 @@ def pgexport_handler(args):
|
|
|
41
42
|
|
|
42
43
|
# Proceed with exporting the table data to CSV (the export settings file is automatically derived)
|
|
43
44
|
# print(f"Starting export of table: {table_name} to CSV file: {export_file if export_file else 'default based on table name'}")
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
try:
|
|
46
|
+
pgexport_manager.export_table(table_name, connection_config, export_file)
|
|
47
|
+
print("Export complete.")
|
|
48
|
+
except ConfigFileCreatedError as e:
|
|
49
|
+
print(e)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pgmonkey.managers.pgimport_manager import PGImportManager
|
|
2
|
+
from pgmonkey.common.exceptions import ConfigFileCreatedError
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
def cli_pgimport_subparser(subparsers):
|
|
@@ -39,6 +40,9 @@ def pgimport_handler(args):
|
|
|
39
40
|
|
|
40
41
|
# Proceed with importing the file (the settings file is automatically derived)
|
|
41
42
|
# print(f"Starting import for file: {import_file} into table: {table_name}")
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
try:
|
|
44
|
+
pgimport_manager.import_file(import_file, table_name, connection_config)
|
|
45
|
+
print("Import complete.")
|
|
46
|
+
except ConfigFileCreatedError as e:
|
|
47
|
+
print(e)
|
|
44
48
|
|
|
@@ -9,6 +9,9 @@ def cli_pg_server_config_subparser(subparsers):
|
|
|
9
9
|
|
|
10
10
|
pg_server_config_parser.add_argument('--filepath', required=True,
|
|
11
11
|
help='Path to the config you want settings generated.')
|
|
12
|
+
pg_server_config_parser.add_argument('--audit', action='store_true', default=False,
|
|
13
|
+
help='Connect to the server and compare current settings '
|
|
14
|
+
'against recommendations.')
|
|
12
15
|
pg_server_config_parser.set_defaults(func=pg_server_config_create_handler,
|
|
13
16
|
pg_server_config_manager=pg_server_config_manager)
|
|
14
17
|
|
|
@@ -16,5 +19,7 @@ def cli_pg_server_config_subparser(subparsers):
|
|
|
16
19
|
def pg_server_config_create_handler(args):
|
|
17
20
|
pg_server_config_manager = args.pg_server_config_manager
|
|
18
21
|
|
|
19
|
-
if args.
|
|
22
|
+
if args.audit:
|
|
23
|
+
pg_server_config_manager.audit_server_config(args.filepath)
|
|
24
|
+
else:
|
|
20
25
|
pg_server_config_manager.get_server_config(args.filepath)
|