pgmonkey 2.3.0__tar.gz → 3.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgmonkey-2.3.0/src/pgmonkey.egg-info → pgmonkey-3.3.0}/PKG-INFO +83 -87
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/README.md +81 -85
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/pyproject.toml +2 -2
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_export_subparser.py +6 -2
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_import_subparser.py +6 -2
- pgmonkey-3.3.0/src/pgmonkey/common/exceptions.py +3 -0
- pgmonkey-3.3.0/src/pgmonkey/common/templates/postgres.yaml +65 -0
- pgmonkey-3.3.0/src/pgmonkey/common/utils/configutils.py +26 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/async_pool_connection.py +6 -3
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/normal_connection.py +3 -1
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/postgres_connection_factory.py +5 -5
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pg_server_config_manager.py +5 -15
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pgcodegen_manager.py +6 -8
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pgconfig_manager.py +4 -7
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pgconnection_manager.py +5 -2
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pgexport_manager.py +1 -2
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/pgimport_manager.py +1 -2
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/serversettings/postgres_server_config_generator.py +9 -7
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/conftest.py +44 -45
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_config_manager.py +5 -5
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_connection_caching.py +20 -24
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_connection_factory.py +17 -17
- pgmonkey-3.3.0/src/pgmonkey/tests/unit/test_csv_data_exporter.py +141 -0
- pgmonkey-3.3.0/src/pgmonkey/tests/unit/test_csv_data_importer.py +311 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_normal_connection.py +7 -1
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_pgconnection_manager.py +1 -1
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_server_config_generator.py +26 -26
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_server_settings_inspector.py +1 -1
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tools/connection_code_generator.py +11 -9
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tools/csv_data_exporter.py +10 -6
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tools/csv_data_importer.py +68 -55
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tools/database_connection_tester.py +3 -1
- {pgmonkey-2.3.0 → pgmonkey-3.3.0/src/pgmonkey.egg-info}/PKG-INFO +83 -87
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey.egg-info/SOURCES.txt +4 -0
- pgmonkey-2.3.0/src/pgmonkey/common/templates/postgres.yaml +0 -66
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/LICENSE +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/NOTICE +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/setup.cfg +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_pg_server_config_subparser.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_pgconfig_subparser.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_settings_subparser.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/cli/cli_toplevel_parser.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/common/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/common/config/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/common/utils/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/common/utils/pathutils.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/base.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/async_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/base_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/pool_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/settings_manager.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/managers/toplevel_manager.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/serversettings/postgres_server_settings_inspector.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/settings/app_settings.yaml +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/integration/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/integration/test_pgconnection_manager_integration.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_async_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_async_pool_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_base_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_code_generator.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_path_utils.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_pool_connection.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tests/unit/test_settings_manager.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/tools/__init__.py +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey.egg-info/dependency_links.txt +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey.egg-info/entry_points.txt +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey.egg-info/requires.txt +0 -0
- {pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgmonkey
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.3.0
|
|
4
4
|
Summary: A tool to assist with postgresql database connections
|
|
5
5
|
Author-email: Good Boy <pythonic@rexbytes.com>
|
|
6
6
|
License: MIT
|
|
@@ -15,7 +15,7 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
15
15
|
Classifier: Operating System :: OS Independent
|
|
16
16
|
Classifier: Topic :: Database
|
|
17
17
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
-
Requires-Python: <
|
|
18
|
+
Requires-Python: <4.0,>=3.10
|
|
19
19
|
Description-Content-Type: text/markdown
|
|
20
20
|
License-File: LICENSE
|
|
21
21
|
License-File: NOTICE
|
|
@@ -90,7 +90,7 @@ pip install pgmonkey[test]
|
|
|
90
90
|
|
|
91
91
|
## One Config File, All Connection Types
|
|
92
92
|
|
|
93
|
-
|
|
93
|
+
pgmonkey uses a **single YAML configuration file** for all connection types. Instead of maintaining separate config files for normal, pool, async, and async_pool connections, you define everything in one file and specify the connection type when you call the API:
|
|
94
94
|
|
|
95
95
|
```python
|
|
96
96
|
from pgmonkey import PGConnectionManager
|
|
@@ -111,55 +111,54 @@ The `connection_type` parameter is optional. If omitted, pgmonkey uses the `conn
|
|
|
111
111
|
Here is the full configuration template. You only need to fill in the sections relevant to the connection types you plan to use.
|
|
112
112
|
|
|
113
113
|
```yaml
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
114
|
+
# Default connection type when none is specified in the API call.
|
|
115
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
116
|
+
# You can override this per-call:
|
|
117
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
118
|
+
connection_type: 'normal'
|
|
119
|
+
|
|
120
|
+
connection_settings:
|
|
121
|
+
user: 'postgres'
|
|
122
|
+
password: 'password'
|
|
123
|
+
host: 'localhost'
|
|
124
|
+
port: '5432'
|
|
125
|
+
dbname: 'mydatabase'
|
|
126
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
127
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
128
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
129
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
130
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
131
|
+
application_name: 'myapp'
|
|
132
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
133
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
134
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
135
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
136
|
+
|
|
137
|
+
# Settings for 'pool' connection type
|
|
138
|
+
pool_settings:
|
|
139
|
+
min_size: 5
|
|
140
|
+
max_size: 20
|
|
141
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
142
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
143
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
144
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
145
|
+
|
|
146
|
+
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
147
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
148
|
+
async_settings:
|
|
149
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
150
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
151
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
152
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
153
|
+
|
|
154
|
+
# Settings for 'async_pool' connection type
|
|
155
|
+
async_pool_settings:
|
|
156
|
+
min_size: 5
|
|
157
|
+
max_size: 20
|
|
158
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
159
|
+
max_idle: 300
|
|
160
|
+
max_lifetime: 3600
|
|
161
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
163
162
|
```
|
|
164
163
|
|
|
165
164
|
### Connection Settings
|
|
@@ -226,13 +225,12 @@ Used by `async_pool` connection type. Same parameters as pool settings. The `asy
|
|
|
226
225
|
The most common method. Credentials are sent to the PostgreSQL server for validation.
|
|
227
226
|
|
|
228
227
|
```yaml
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
dbname: 'your_database'
|
|
228
|
+
connection_type: 'normal'
|
|
229
|
+
connection_settings:
|
|
230
|
+
user: 'your_user'
|
|
231
|
+
password: 'your_password'
|
|
232
|
+
host: 'localhost'
|
|
233
|
+
dbname: 'your_database'
|
|
236
234
|
```
|
|
237
235
|
|
|
238
236
|
### SSL/TLS Encryption
|
|
@@ -247,15 +245,14 @@ SSL/TLS encrypts the connection between your application and the PostgreSQL serv
|
|
|
247
245
|
- `verify-full`: Require SSL, verify certificate, and ensure the hostname matches.
|
|
248
246
|
|
|
249
247
|
```yaml
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
sslrootcert: '/path/to/ca.crt'
|
|
248
|
+
connection_type: 'normal'
|
|
249
|
+
connection_settings:
|
|
250
|
+
user: 'your_user'
|
|
251
|
+
password: 'your_password'
|
|
252
|
+
host: 'localhost'
|
|
253
|
+
dbname: 'your_database'
|
|
254
|
+
sslmode: 'require'
|
|
255
|
+
sslrootcert: '/path/to/ca.crt'
|
|
259
256
|
```
|
|
260
257
|
|
|
261
258
|
### Certificate-Based Authentication
|
|
@@ -263,17 +260,16 @@ postgresql:
|
|
|
263
260
|
Uses SSL client certificates for authentication. Highly secure and often used in enterprise environments.
|
|
264
261
|
|
|
265
262
|
```yaml
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
sslrootcert: '/path/to/ca.crt'
|
|
263
|
+
connection_type: 'normal'
|
|
264
|
+
connection_settings:
|
|
265
|
+
user: 'your_user'
|
|
266
|
+
password: 'your_password'
|
|
267
|
+
host: 'localhost'
|
|
268
|
+
dbname: 'your_database'
|
|
269
|
+
sslmode: 'verify-full'
|
|
270
|
+
sslcert: '/path/to/client.crt'
|
|
271
|
+
sslkey: '/path/to/client.key'
|
|
272
|
+
sslrootcert: '/path/to/ca.crt'
|
|
277
273
|
```
|
|
278
274
|
|
|
279
275
|
## Using the CLI
|
|
@@ -326,8 +322,8 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-typ
|
|
|
326
322
|
|
|
327
323
|
The `--library` flag controls which library the generated code targets:
|
|
328
324
|
|
|
329
|
-
- `pgmonkey` (default)
|
|
330
|
-
- `psycopg`
|
|
325
|
+
- `pgmonkey` (default) - generates code using pgmonkey's `PGConnectionManager`.
|
|
326
|
+
- `psycopg` - generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
331
327
|
|
|
332
328
|
### Server Configuration Recommendations
|
|
333
329
|
|
|
@@ -386,7 +382,7 @@ This queries the server's `pg_settings` (read-only) and displays a comparison ta
|
|
|
386
382
|
|
|
387
383
|
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
388
384
|
|
|
389
|
-
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified
|
|
385
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified - the audit is entirely read-only.
|
|
390
386
|
|
|
391
387
|
### Importing and Exporting Data
|
|
392
388
|
|
|
@@ -538,11 +534,11 @@ except Exception as e:
|
|
|
538
534
|
|
|
539
535
|
pgmonkey handles several production concerns behind the scenes so you don't have to:
|
|
540
536
|
|
|
541
|
-
- **Connection caching**
|
|
542
|
-
- **Async pool lifecycle**
|
|
543
|
-
- **atexit cleanup**
|
|
544
|
-
- **Thread-safe caching**
|
|
545
|
-
- **Config validation**
|
|
537
|
+
- **Connection caching** - Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
538
|
+
- **Async pool lifecycle** - `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
539
|
+
- **atexit cleanup** - All cached connections are automatically closed when the process exits.
|
|
540
|
+
- **Thread-safe caching** - The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
541
|
+
- **Config validation** - Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
546
542
|
|
|
547
543
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
548
544
|
|
|
@@ -58,7 +58,7 @@ pip install pgmonkey[test]
|
|
|
58
58
|
|
|
59
59
|
## One Config File, All Connection Types
|
|
60
60
|
|
|
61
|
-
|
|
61
|
+
pgmonkey uses a **single YAML configuration file** for all connection types. Instead of maintaining separate config files for normal, pool, async, and async_pool connections, you define everything in one file and specify the connection type when you call the API:
|
|
62
62
|
|
|
63
63
|
```python
|
|
64
64
|
from pgmonkey import PGConnectionManager
|
|
@@ -79,55 +79,54 @@ The `connection_type` parameter is optional. If omitted, pgmonkey uses the `conn
|
|
|
79
79
|
Here is the full configuration template. You only need to fill in the sections relevant to the connection types you plan to use.
|
|
80
80
|
|
|
81
81
|
```yaml
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
82
|
+
# Default connection type when none is specified in the API call.
|
|
83
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
84
|
+
# You can override this per-call:
|
|
85
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
86
|
+
connection_type: 'normal'
|
|
87
|
+
|
|
88
|
+
connection_settings:
|
|
89
|
+
user: 'postgres'
|
|
90
|
+
password: 'password'
|
|
91
|
+
host: 'localhost'
|
|
92
|
+
port: '5432'
|
|
93
|
+
dbname: 'mydatabase'
|
|
94
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
95
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
96
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
97
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
98
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
99
|
+
application_name: 'myapp'
|
|
100
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
101
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
102
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
103
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
104
|
+
|
|
105
|
+
# Settings for 'pool' connection type
|
|
106
|
+
pool_settings:
|
|
107
|
+
min_size: 5
|
|
108
|
+
max_size: 20
|
|
109
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
110
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
111
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
112
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
113
|
+
|
|
114
|
+
# Settings for 'async' connection type (applied via SET commands on connection)
|
|
115
|
+
# These settings are also applied to 'async_pool' connections via a configure callback.
|
|
116
|
+
async_settings:
|
|
117
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
118
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
119
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
120
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
121
|
+
|
|
122
|
+
# Settings for 'async_pool' connection type
|
|
123
|
+
async_pool_settings:
|
|
124
|
+
min_size: 5
|
|
125
|
+
max_size: 20
|
|
126
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
127
|
+
max_idle: 300
|
|
128
|
+
max_lifetime: 3600
|
|
129
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
131
130
|
```
|
|
132
131
|
|
|
133
132
|
### Connection Settings
|
|
@@ -194,13 +193,12 @@ Used by `async_pool` connection type. Same parameters as pool settings. The `asy
|
|
|
194
193
|
The most common method. Credentials are sent to the PostgreSQL server for validation.
|
|
195
194
|
|
|
196
195
|
```yaml
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
dbname: 'your_database'
|
|
196
|
+
connection_type: 'normal'
|
|
197
|
+
connection_settings:
|
|
198
|
+
user: 'your_user'
|
|
199
|
+
password: 'your_password'
|
|
200
|
+
host: 'localhost'
|
|
201
|
+
dbname: 'your_database'
|
|
204
202
|
```
|
|
205
203
|
|
|
206
204
|
### SSL/TLS Encryption
|
|
@@ -215,15 +213,14 @@ SSL/TLS encrypts the connection between your application and the PostgreSQL serv
|
|
|
215
213
|
- `verify-full`: Require SSL, verify certificate, and ensure the hostname matches.
|
|
216
214
|
|
|
217
215
|
```yaml
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
sslrootcert: '/path/to/ca.crt'
|
|
216
|
+
connection_type: 'normal'
|
|
217
|
+
connection_settings:
|
|
218
|
+
user: 'your_user'
|
|
219
|
+
password: 'your_password'
|
|
220
|
+
host: 'localhost'
|
|
221
|
+
dbname: 'your_database'
|
|
222
|
+
sslmode: 'require'
|
|
223
|
+
sslrootcert: '/path/to/ca.crt'
|
|
227
224
|
```
|
|
228
225
|
|
|
229
226
|
### Certificate-Based Authentication
|
|
@@ -231,17 +228,16 @@ postgresql:
|
|
|
231
228
|
Uses SSL client certificates for authentication. Highly secure and often used in enterprise environments.
|
|
232
229
|
|
|
233
230
|
```yaml
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
sslrootcert: '/path/to/ca.crt'
|
|
231
|
+
connection_type: 'normal'
|
|
232
|
+
connection_settings:
|
|
233
|
+
user: 'your_user'
|
|
234
|
+
password: 'your_password'
|
|
235
|
+
host: 'localhost'
|
|
236
|
+
dbname: 'your_database'
|
|
237
|
+
sslmode: 'verify-full'
|
|
238
|
+
sslcert: '/path/to/client.crt'
|
|
239
|
+
sslkey: '/path/to/client.key'
|
|
240
|
+
sslrootcert: '/path/to/ca.crt'
|
|
245
241
|
```
|
|
246
242
|
|
|
247
243
|
## Using the CLI
|
|
@@ -294,8 +290,8 @@ pgmonkey pgconfig generate-code --filepath /path/to/config.yaml --connection-typ
|
|
|
294
290
|
|
|
295
291
|
The `--library` flag controls which library the generated code targets:
|
|
296
292
|
|
|
297
|
-
- `pgmonkey` (default)
|
|
298
|
-
- `psycopg`
|
|
293
|
+
- `pgmonkey` (default) - generates code using pgmonkey's `PGConnectionManager`.
|
|
294
|
+
- `psycopg` - generates code using `psycopg` and `psycopg_pool` directly, reading connection settings from the same YAML config file.
|
|
299
295
|
|
|
300
296
|
### Server Configuration Recommendations
|
|
301
297
|
|
|
@@ -354,7 +350,7 @@ This queries the server's `pg_settings` (read-only) and displays a comparison ta
|
|
|
354
350
|
|
|
355
351
|
The audit also inspects `pg_hba_file_rules` (PostgreSQL 15+) when available, showing current HBA rules alongside recommendations.
|
|
356
352
|
|
|
357
|
-
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified
|
|
353
|
+
If the connected role lacks permission to query `pg_settings`, the audit fails gracefully with a message and falls back to showing recommendations only. No server settings are ever modified - the audit is entirely read-only.
|
|
358
354
|
|
|
359
355
|
### Importing and Exporting Data
|
|
360
356
|
|
|
@@ -506,11 +502,11 @@ except Exception as e:
|
|
|
506
502
|
|
|
507
503
|
pgmonkey handles several production concerns behind the scenes so you don't have to:
|
|
508
504
|
|
|
509
|
-
- **Connection caching**
|
|
510
|
-
- **Async pool lifecycle**
|
|
511
|
-
- **atexit cleanup**
|
|
512
|
-
- **Thread-safe caching**
|
|
513
|
-
- **Config validation**
|
|
505
|
+
- **Connection caching** - Connections and pools are cached by config content (SHA-256 hash). Repeated calls with the same config return the existing instance, preventing "pool storms" where each call opens a new pool.
|
|
506
|
+
- **Async pool lifecycle** - `async with pool_conn:` borrows a connection from the pool and returns it when the block exits. The pool stays open for reuse. Auto-commits on clean exit, rolls back on exception.
|
|
507
|
+
- **atexit cleanup** - All cached connections are automatically closed when the process exits.
|
|
508
|
+
- **Thread-safe caching** - The connection cache is protected by a threading lock with double-check locking to prevent race conditions.
|
|
509
|
+
- **Config validation** - Unknown connection setting keys produce a warning log message. Pool settings are validated (e.g., `min_size` cannot exceed `max_size`).
|
|
514
510
|
|
|
515
511
|
### App-Level Pattern: Sync Database Class (Flask)
|
|
516
512
|
|
|
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pgmonkey"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.3.0"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name="Good Boy", email="pythonic@rexbytes.com" },
|
|
10
10
|
]
|
|
11
11
|
description = "A tool to assist with postgresql database connections"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = {text = "MIT"}
|
|
14
|
-
requires-python = ">=3.10,<
|
|
14
|
+
requires-python = ">=3.10,<4.0"
|
|
15
15
|
classifiers = [
|
|
16
16
|
"Programming Language :: Python :: 3.10",
|
|
17
17
|
"Programming Language :: Python :: 3.11",
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pgmonkey.managers.pgexport_manager import PGExportManager
|
|
2
|
+
from pgmonkey.common.exceptions import ConfigFileCreatedError
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
|
|
@@ -41,5 +42,8 @@ def pgexport_handler(args):
|
|
|
41
42
|
|
|
42
43
|
# Proceed with exporting the table data to CSV (the export settings file is automatically derived)
|
|
43
44
|
# print(f"Starting export of table: {table_name} to CSV file: {export_file if export_file else 'default based on table name'}")
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
try:
|
|
46
|
+
pgexport_manager.export_table(table_name, connection_config, export_file)
|
|
47
|
+
print("Export complete.")
|
|
48
|
+
except ConfigFileCreatedError as e:
|
|
49
|
+
print(e)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pgmonkey.managers.pgimport_manager import PGImportManager
|
|
2
|
+
from pgmonkey.common.exceptions import ConfigFileCreatedError
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
def cli_pgimport_subparser(subparsers):
|
|
@@ -39,6 +40,9 @@ def pgimport_handler(args):
|
|
|
39
40
|
|
|
40
41
|
# Proceed with importing the file (the settings file is automatically derived)
|
|
41
42
|
# print(f"Starting import for file: {import_file} into table: {table_name}")
|
|
42
|
-
|
|
43
|
-
|
|
43
|
+
try:
|
|
44
|
+
pgimport_manager.import_file(import_file, table_name, connection_config)
|
|
45
|
+
print("Import complete.")
|
|
46
|
+
except ConfigFileCreatedError as e:
|
|
47
|
+
print(e)
|
|
44
48
|
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Default connection type used when none is specified in the API call.
|
|
2
|
+
# Options: 'normal', 'pool', 'async', 'async_pool'
|
|
3
|
+
# You can override this per-call:
|
|
4
|
+
# manager.get_database_connection('config.yaml', 'pool')
|
|
5
|
+
connection_type: 'normal'
|
|
6
|
+
|
|
7
|
+
connection_settings:
|
|
8
|
+
user: 'postgres'
|
|
9
|
+
password: 'password'
|
|
10
|
+
host: 'localhost'
|
|
11
|
+
port: '5432'
|
|
12
|
+
dbname: 'mydatabase'
|
|
13
|
+
sslmode: 'prefer' # Options: disable, allow, prefer, require, verify-ca, verify-full
|
|
14
|
+
sslcert: '' # Path to the client SSL certificate, if needed
|
|
15
|
+
sslkey: '' # Path to the client SSL key, if needed
|
|
16
|
+
sslrootcert: '' # Path to the root SSL certificate, if needed
|
|
17
|
+
connect_timeout: '10' # Maximum wait for connection, in seconds
|
|
18
|
+
application_name: 'myapp'
|
|
19
|
+
keepalives: '1' # Enable TCP keepalives (1=on, 0=off)
|
|
20
|
+
keepalives_idle: '60' # Seconds before sending a keepalive probe
|
|
21
|
+
keepalives_interval: '15' # Seconds between keepalive probes
|
|
22
|
+
keepalives_count: '5' # Max keepalive probes before closing the connection
|
|
23
|
+
# autocommit: false # Enable autocommit mode (required for DDL like CREATE DATABASE, VACUUM, etc.)
|
|
24
|
+
|
|
25
|
+
# Session-level GUC settings for 'normal' and 'pool' connection types.
|
|
26
|
+
# For 'normal': applied via SET commands after connect.
|
|
27
|
+
# For 'pool': applied via psycopg_pool's configure callback on each checkout.
|
|
28
|
+
sync_settings:
|
|
29
|
+
# statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
30
|
+
# lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
31
|
+
# idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
32
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
33
|
+
|
|
34
|
+
# Settings for 'pool' connection type
|
|
35
|
+
pool_settings:
|
|
36
|
+
min_size: 5
|
|
37
|
+
max_size: 20
|
|
38
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
39
|
+
max_idle: 300 # Seconds a connection can remain idle before being closed
|
|
40
|
+
max_lifetime: 3600 # Seconds a connection can be reused
|
|
41
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
42
|
+
# max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
|
|
43
|
+
# reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
|
|
44
|
+
# num_workers: 3 # Number of background workers maintaining pool connections
|
|
45
|
+
|
|
46
|
+
# Session-level GUC settings for 'async' and 'async_pool' connection types.
|
|
47
|
+
# For 'async': applied via SET commands after connect.
|
|
48
|
+
# For 'async_pool': applied via psycopg_pool's configure callback on each checkout.
|
|
49
|
+
async_settings:
|
|
50
|
+
idle_in_transaction_session_timeout: '5000' # Timeout for idle in transaction (ms)
|
|
51
|
+
statement_timeout: '30000' # Cancel statements exceeding this time (ms)
|
|
52
|
+
lock_timeout: '10000' # Timeout for acquiring locks (ms)
|
|
53
|
+
# work_mem: '256MB' # Memory for sort operations and more
|
|
54
|
+
|
|
55
|
+
# Settings for 'async_pool' connection type
|
|
56
|
+
async_pool_settings:
|
|
57
|
+
min_size: 5
|
|
58
|
+
max_size: 20
|
|
59
|
+
timeout: 30 # Seconds to wait for a connection from the pool before raising an error
|
|
60
|
+
max_idle: 300
|
|
61
|
+
max_lifetime: 3600
|
|
62
|
+
check_on_checkout: false # Validate connections with SELECT 1 before handing to caller
|
|
63
|
+
# max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
|
|
64
|
+
# reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
|
|
65
|
+
# num_workers: 3 # Number of background workers maintaining pool connections
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def normalize_config(config_data):
|
|
5
|
+
"""Normalize a pgmonkey configuration dictionary.
|
|
6
|
+
|
|
7
|
+
In pgmonkey v3.0.0, the top-level 'postgresql:' wrapper key was removed.
|
|
8
|
+
This function detects the old format and unwraps it with a deprecation warning.
|
|
9
|
+
|
|
10
|
+
Args:
|
|
11
|
+
config_data: Configuration dictionary (old or new format).
|
|
12
|
+
|
|
13
|
+
Returns:
|
|
14
|
+
The normalized configuration dictionary (without the 'postgresql:' wrapper).
|
|
15
|
+
"""
|
|
16
|
+
if isinstance(config_data, dict) and 'postgresql' in config_data:
|
|
17
|
+
warnings.warn(
|
|
18
|
+
"The top-level 'postgresql:' key in pgmonkey config files is deprecated "
|
|
19
|
+
"since v3.0.0 and will be removed in a future version. "
|
|
20
|
+
"Remove the 'postgresql:' wrapper and dedent all settings one level. "
|
|
21
|
+
"See https://pgmonkey.net/reference.html for the new format.",
|
|
22
|
+
DeprecationWarning,
|
|
23
|
+
stacklevel=3,
|
|
24
|
+
)
|
|
25
|
+
return config_data['postgresql']
|
|
26
|
+
return config_data
|
{pgmonkey-2.3.0 → pgmonkey-3.3.0}/src/pgmonkey/connections/postgres/async_pool_connection.py
RENAMED
|
@@ -8,8 +8,6 @@ from contextlib import asynccontextmanager
|
|
|
8
8
|
|
|
9
9
|
logger = logging.getLogger(__name__)
|
|
10
10
|
|
|
11
|
-
warnings.filterwarnings('ignore', category=RuntimeWarning, module='psycopg_pool')
|
|
12
|
-
|
|
13
11
|
|
|
14
12
|
class PGAsyncPoolConnection(PostgresBaseConnection):
|
|
15
13
|
def __init__(self, config, async_pool_settings=None, async_settings=None):
|
|
@@ -47,7 +45,12 @@ class PGAsyncPoolConnection(PostgresBaseConnection):
|
|
|
47
45
|
logger.warning("Could not apply setting '%s': %s", setting, e)
|
|
48
46
|
kwargs['configure'] = _configure
|
|
49
47
|
|
|
50
|
-
|
|
48
|
+
# Suppress RuntimeWarnings that psycopg_pool may emit during
|
|
49
|
+
# pool construction. Scoped to construction only so that
|
|
50
|
+
# warnings during normal pool operation remain visible.
|
|
51
|
+
with warnings.catch_warnings():
|
|
52
|
+
warnings.filterwarnings('ignore', category=RuntimeWarning, module='psycopg_pool')
|
|
53
|
+
self.pool = AsyncConnectionPool(conninfo=conninfo, **kwargs)
|
|
51
54
|
await self.pool.open()
|
|
52
55
|
|
|
53
56
|
async def test_connection(self):
|