pgmonkey 3.2.0__tar.gz → 3.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgmonkey-3.2.0/src/pgmonkey.egg-info → pgmonkey-3.4.0}/PKG-INFO +213 -7
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/README.md +212 -6
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/pyproject.toml +1 -1
- pgmonkey-3.4.0/src/pgmonkey/__init__.py +3 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_pgconfig_subparser.py +12 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/templates/postgres.yaml +18 -0
- pgmonkey-3.4.0/src/pgmonkey/common/utils/configutils.py +71 -0
- pgmonkey-3.4.0/src/pgmonkey/common/utils/envutils.py +212 -0
- pgmonkey-3.4.0/src/pgmonkey/common/utils/redaction.py +49 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgcodegen_manager.py +4 -1
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgconfig_manager.py +5 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgconnection_manager.py +15 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgexport_manager.py +1 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgimport_manager.py +1 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_csv_data_exporter.py +9 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_csv_data_importer.py +118 -0
- pgmonkey-3.4.0/src/pgmonkey/tests/unit/test_env_interpolation.py +490 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/csv_data_exporter.py +1 -1
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/csv_data_importer.py +9 -2
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/database_connection_tester.py +19 -7
- {pgmonkey-3.2.0 → pgmonkey-3.4.0/src/pgmonkey.egg-info}/PKG-INFO +213 -7
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/SOURCES.txt +3 -0
- pgmonkey-3.2.0/src/pgmonkey/__init__.py +0 -1
- pgmonkey-3.2.0/src/pgmonkey/common/utils/configutils.py +0 -26
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/LICENSE +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/NOTICE +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/setup.cfg +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_export_subparser.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_import_subparser.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_pg_server_config_subparser.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_settings_subparser.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_toplevel_parser.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/config/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/exceptions.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/utils/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/utils/pathutils.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/base.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/async_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/async_pool_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/base_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/normal_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/pool_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/postgres_connection_factory.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pg_server_config_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/settings_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/toplevel_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/serversettings/postgres_server_config_generator.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/serversettings/postgres_server_settings_inspector.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/settings/app_settings.yaml +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/conftest.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/integration/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/integration/test_pgconnection_manager_integration.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_async_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_async_pool_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_base_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_code_generator.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_config_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_connection_caching.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_connection_factory.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_normal_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_path_utils.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_pgconnection_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_pool_connection.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_server_config_generator.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_server_settings_inspector.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_settings_manager.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/__init__.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/connection_code_generator.py +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/dependency_links.txt +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/entry_points.txt +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/requires.txt +0 -0
- {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pgmonkey
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.0
|
|
4
4
|
Summary: A tool to assist with postgresql database connections
|
|
5
5
|
Author-email: Good Boy <pythonic@rexbytes.com>
|
|
6
6
|
License: MIT
|
|
@@ -47,24 +47,33 @@ Dynamic: license-file
|
|
|
47
47
|
- [Password-Based Authentication](#password-based-authentication)
|
|
48
48
|
- [SSL/TLS Encryption](#ssltls-encryption)
|
|
49
49
|
- [Certificate-Based Authentication](#certificate-based-authentication)
|
|
50
|
-
5. [
|
|
50
|
+
5. [Environment Variable Interpolation (Advanced)](#environment-variable-interpolation-advanced)
|
|
51
|
+
- [Standard YAML (Default)](#standard-yaml-default)
|
|
52
|
+
- [Inline Syntax: ${VAR}](#inline-syntax-var)
|
|
53
|
+
- [Structured Syntax: from_env / from_file](#structured-syntax-from_env--from_file)
|
|
54
|
+
- [Sensitive Key Protection](#sensitive-key-protection)
|
|
55
|
+
- [Python API](#python-api)
|
|
56
|
+
- [CLI Usage](#cli-usage)
|
|
57
|
+
- [Redacting Secrets for Logging](#redacting-secrets-for-logging)
|
|
58
|
+
- [Deployment Patterns](#deployment-patterns)
|
|
59
|
+
6. [Using the CLI](#using-the-cli)
|
|
51
60
|
- [Creating a Configuration Template](#creating-a-configuration-template)
|
|
52
61
|
- [Testing a Connection](#testing-a-connection)
|
|
53
62
|
- [Generating Python Code](#generating-python-code)
|
|
54
63
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
55
64
|
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
56
65
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
57
|
-
|
|
66
|
+
7. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
58
67
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
59
68
|
- [Pooled Connection](#pooled-connection)
|
|
60
69
|
- [Async Connection](#async-connection)
|
|
61
70
|
- [Async Pooled Connection](#async-pooled-connection)
|
|
62
71
|
- [Using the Config File Default](#using-the-config-file-default)
|
|
63
72
|
- [Transactions, Commit, and Rollback](#transactions-commit-and-rollback)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
73
|
+
8. [Best Practice Recipes](#best-practice-recipes)
|
|
74
|
+
9. [Testing All Connection Types](#testing-all-connection-types)
|
|
75
|
+
10. [Testing Pool Capacity](#testing-pool-capacity)
|
|
76
|
+
11. [Running the Test Suite](#running-the-test-suite)
|
|
68
77
|
|
|
69
78
|
## Installation
|
|
70
79
|
|
|
@@ -272,6 +281,203 @@ connection_settings:
|
|
|
272
281
|
sslrootcert: '/path/to/ca.crt'
|
|
273
282
|
```
|
|
274
283
|
|
|
284
|
+
## Environment Variable Interpolation (Advanced)
|
|
285
|
+
|
|
286
|
+
pgmonkey v3.4.0 adds opt-in support for resolving environment variables and secret file references inside YAML configuration files. This lets you keep your config files free of hardcoded credentials while staying compatible with standard deployment workflows (12-factor env vars, Docker, Kubernetes).
|
|
287
|
+
|
|
288
|
+
**Interpolation is disabled by default.** If you do not enable it, pgmonkey treats every YAML value as a literal string - exactly as it always has.
|
|
289
|
+
|
|
290
|
+
### Standard YAML (Default)
|
|
291
|
+
|
|
292
|
+
The standard approach is to write literal values directly in the config file. This is the simplest way to get started and requires no extra flags or API parameters:
|
|
293
|
+
|
|
294
|
+
```yaml
|
|
295
|
+
connection_type: 'normal'
|
|
296
|
+
|
|
297
|
+
connection_settings:
|
|
298
|
+
user: 'postgres'
|
|
299
|
+
password: 'my_password'
|
|
300
|
+
host: 'localhost'
|
|
301
|
+
port: '5432'
|
|
302
|
+
dbname: 'mydatabase'
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
from pgmonkey import PGConnectionManager
|
|
307
|
+
|
|
308
|
+
manager = PGConnectionManager()
|
|
309
|
+
conn = manager.get_database_connection('config.yaml')
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
This continues to work exactly as before. No changes needed for existing configs.
|
|
313
|
+
|
|
314
|
+
### Inline Syntax: ${VAR}
|
|
315
|
+
|
|
316
|
+
When interpolation is enabled, you can reference environment variables using `${VAR}` syntax. Use `${VAR:-default}` to provide a fallback value when the variable is not set:
|
|
317
|
+
|
|
318
|
+
```yaml
|
|
319
|
+
connection_type: 'normal'
|
|
320
|
+
|
|
321
|
+
connection_settings:
|
|
322
|
+
user: '${PGUSER:-postgres}'
|
|
323
|
+
password: '${PGPASSWORD}'
|
|
324
|
+
host: '${PGHOST:-localhost}'
|
|
325
|
+
port: '${PGPORT:-5432}'
|
|
326
|
+
dbname: '${PGDATABASE:-mydb}'
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
Rules:
|
|
330
|
+
- If a referenced variable is **not set** and **no default** is provided, pgmonkey raises `EnvInterpolationError` with a clear message naming the variable and the config key.
|
|
331
|
+
- Multiple references can appear in a single value: `'${PGHOST}:${PGPORT}'`
|
|
332
|
+
- Non-string values (integers, booleans) pass through unchanged.
|
|
333
|
+
|
|
334
|
+
### Structured Syntax: from_env / from_file
|
|
335
|
+
|
|
336
|
+
For secrets, pgmonkey supports a structured YAML form that makes the intent unambiguous:
|
|
337
|
+
|
|
338
|
+
**Read from an environment variable:**
|
|
339
|
+
|
|
340
|
+
```yaml
|
|
341
|
+
connection_settings:
|
|
342
|
+
password:
|
|
343
|
+
from_env: PGMONKEY_DB_PASSWORD
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
**Read from a file (Kubernetes Secret-style):**
|
|
347
|
+
|
|
348
|
+
```yaml
|
|
349
|
+
connection_settings:
|
|
350
|
+
password:
|
|
351
|
+
from_file: /var/run/secrets/db/password
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
Rules:
|
|
355
|
+
- `from_env` reads the named environment variable. If it is not set, pgmonkey raises `EnvInterpolationError`.
|
|
356
|
+
- `from_file` reads the file contents and trims the trailing newline (matching Kubernetes Secret conventions). If the file does not exist or cannot be read, pgmonkey raises `EnvInterpolationError`.
|
|
357
|
+
- A structured reference must be a dict with exactly one key (`from_env` or `from_file`). Dicts with additional keys are treated as normal nested config.
|
|
358
|
+
|
|
359
|
+
### Sensitive Key Protection
|
|
360
|
+
|
|
361
|
+
By default, `${VAR:-default}` fallback values are **disallowed** for sensitive keys. This prevents accidentally shipping a config with a hardcoded fallback password that silently takes over when the env var is missing.
|
|
362
|
+
|
|
363
|
+
Sensitive keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`.
|
|
364
|
+
|
|
365
|
+
```yaml
|
|
366
|
+
# This will FAIL (password is a sensitive key, defaults not allowed):
|
|
367
|
+
password: '${PGPASSWORD:-fallback_password}'
|
|
368
|
+
|
|
369
|
+
# This is fine (host is not sensitive):
|
|
370
|
+
host: '${PGHOST:-localhost}'
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
To explicitly allow sensitive defaults (e.g. for local development), pass `allow_sensitive_defaults=True`:
|
|
374
|
+
|
|
375
|
+
```python
|
|
376
|
+
cfg = load_config('config.yaml', resolve_env=True, allow_sensitive_defaults=True)
|
|
377
|
+
```
|
|
378
|
+
|
|
379
|
+
### Python API
|
|
380
|
+
|
|
381
|
+
Use `load_config()` for the simplest path to a resolved config dictionary:
|
|
382
|
+
|
|
383
|
+
```python
|
|
384
|
+
from pgmonkey import load_config
|
|
385
|
+
|
|
386
|
+
# Load without interpolation (default - same as always)
|
|
387
|
+
cfg = load_config('config.yaml')
|
|
388
|
+
|
|
389
|
+
# Load with env interpolation enabled
|
|
390
|
+
cfg = load_config('config.yaml', resolve_env=True)
|
|
391
|
+
|
|
392
|
+
# Strict mode: fail on missing vars
|
|
393
|
+
cfg = load_config('config.yaml', resolve_env=True, strict=True)
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
Or pass `resolve_env=True` directly to the connection manager:
|
|
397
|
+
|
|
398
|
+
```python
|
|
399
|
+
from pgmonkey import PGConnectionManager
|
|
400
|
+
|
|
401
|
+
manager = PGConnectionManager()
|
|
402
|
+
conn = manager.get_database_connection('config.yaml', resolve_env=True)
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
Both `get_database_connection()` and `get_database_connection_from_dict()` accept the `resolve_env` parameter.
|
|
406
|
+
|
|
407
|
+
### CLI Usage
|
|
408
|
+
|
|
409
|
+
Add `--resolve-env` to any `pgconfig test` or `pgconfig generate-code` command:
|
|
410
|
+
|
|
411
|
+
```bash
|
|
412
|
+
# Test a connection with env vars resolved
|
|
413
|
+
pgmonkey pgconfig test --connconfig config.yaml --resolve-env
|
|
414
|
+
|
|
415
|
+
# Generate code (--resolve-env accepted for consistency)
|
|
416
|
+
pgmonkey pgconfig generate-code --connconfig config.yaml --resolve-env
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
Without `--resolve-env`, the CLI works exactly as before - `${VAR}` patterns are treated as literal strings.
|
|
420
|
+
|
|
421
|
+
### Redacting Secrets for Logging
|
|
422
|
+
|
|
423
|
+
pgmonkey includes a `redact_config()` utility that replaces sensitive values with `***REDACTED***`, safe for logging or printing:
|
|
424
|
+
|
|
425
|
+
```python
|
|
426
|
+
from pgmonkey import load_config
|
|
427
|
+
from pgmonkey.common.utils.redaction import redact_config
|
|
428
|
+
|
|
429
|
+
cfg = load_config('config.yaml', resolve_env=True)
|
|
430
|
+
print(redact_config(cfg))
|
|
431
|
+
# {'connection_settings': {'password': '***REDACTED***', 'host': 'db.prod.com', ...}}
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
Redacted keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`. Empty strings and `None` values are left as-is (nothing to leak).
|
|
435
|
+
|
|
436
|
+
### Deployment Patterns
|
|
437
|
+
|
|
438
|
+
**Local development** - set env vars in your shell or a `.env` file (managed by your own tooling):
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
export PGPASSWORD=dev_password
|
|
442
|
+
export PGHOST=localhost
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
**Docker / containers** - pass env vars via `docker run -e` or `docker-compose.yml`:
|
|
446
|
+
|
|
447
|
+
```yaml
|
|
448
|
+
# docker-compose.yml
|
|
449
|
+
services:
|
|
450
|
+
app:
|
|
451
|
+
environment:
|
|
452
|
+
PGPASSWORD: ${DB_PASSWORD}
|
|
453
|
+
PGHOST: db
|
|
454
|
+
```
|
|
455
|
+
|
|
456
|
+
**Kubernetes** - mount secrets as files and use `from_file`:
|
|
457
|
+
|
|
458
|
+
```yaml
|
|
459
|
+
# pgmonkey config
|
|
460
|
+
connection_settings:
|
|
461
|
+
password:
|
|
462
|
+
from_file: /var/run/secrets/db/password
|
|
463
|
+
host: '${PGHOST:-db-service}'
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
```yaml
|
|
467
|
+
# k8s pod spec
|
|
468
|
+
volumes:
|
|
469
|
+
- name: db-secret
|
|
470
|
+
secret:
|
|
471
|
+
secretName: db-credentials
|
|
472
|
+
containers:
|
|
473
|
+
- volumeMounts:
|
|
474
|
+
- name: db-secret
|
|
475
|
+
mountPath: /var/run/secrets/db
|
|
476
|
+
readOnly: true
|
|
477
|
+
```
|
|
478
|
+
|
|
479
|
+
**Cloud secret managers** (AWS Secrets Manager, GCP Secret Manager, HashiCorp Vault) - resolve secrets in your deployment pipeline and set them as env vars. pgmonkey does not integrate with cloud vaults directly.
|
|
480
|
+
|
|
275
481
|
## Using the CLI
|
|
276
482
|
|
|
277
483
|
pgmonkey provides a command-line interface for managing configurations and connections.
|
|
@@ -15,24 +15,33 @@
|
|
|
15
15
|
- [Password-Based Authentication](#password-based-authentication)
|
|
16
16
|
- [SSL/TLS Encryption](#ssltls-encryption)
|
|
17
17
|
- [Certificate-Based Authentication](#certificate-based-authentication)
|
|
18
|
-
5. [
|
|
18
|
+
5. [Environment Variable Interpolation (Advanced)](#environment-variable-interpolation-advanced)
|
|
19
|
+
- [Standard YAML (Default)](#standard-yaml-default)
|
|
20
|
+
- [Inline Syntax: ${VAR}](#inline-syntax-var)
|
|
21
|
+
- [Structured Syntax: from_env / from_file](#structured-syntax-from_env--from_file)
|
|
22
|
+
- [Sensitive Key Protection](#sensitive-key-protection)
|
|
23
|
+
- [Python API](#python-api)
|
|
24
|
+
- [CLI Usage](#cli-usage)
|
|
25
|
+
- [Redacting Secrets for Logging](#redacting-secrets-for-logging)
|
|
26
|
+
- [Deployment Patterns](#deployment-patterns)
|
|
27
|
+
6. [Using the CLI](#using-the-cli)
|
|
19
28
|
- [Creating a Configuration Template](#creating-a-configuration-template)
|
|
20
29
|
- [Testing a Connection](#testing-a-connection)
|
|
21
30
|
- [Generating Python Code](#generating-python-code)
|
|
22
31
|
- [Server Configuration Recommendations](#server-configuration-recommendations)
|
|
23
32
|
- [Auditing Live Server Settings](#auditing-live-server-settings)
|
|
24
33
|
- [Importing and Exporting Data](#importing-and-exporting-data)
|
|
25
|
-
|
|
34
|
+
7. [Using pgmonkey in Python](#using-pgmonkey-in-python)
|
|
26
35
|
- [Normal (Synchronous) Connection](#normal-synchronous-connection)
|
|
27
36
|
- [Pooled Connection](#pooled-connection)
|
|
28
37
|
- [Async Connection](#async-connection)
|
|
29
38
|
- [Async Pooled Connection](#async-pooled-connection)
|
|
30
39
|
- [Using the Config File Default](#using-the-config-file-default)
|
|
31
40
|
- [Transactions, Commit, and Rollback](#transactions-commit-and-rollback)
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
41
|
+
8. [Best Practice Recipes](#best-practice-recipes)
|
|
42
|
+
9. [Testing All Connection Types](#testing-all-connection-types)
|
|
43
|
+
10. [Testing Pool Capacity](#testing-pool-capacity)
|
|
44
|
+
11. [Running the Test Suite](#running-the-test-suite)
|
|
36
45
|
|
|
37
46
|
## Installation
|
|
38
47
|
|
|
@@ -240,6 +249,203 @@ connection_settings:
|
|
|
240
249
|
sslrootcert: '/path/to/ca.crt'
|
|
241
250
|
```
|
|
242
251
|
|
|
252
|
+
## Environment Variable Interpolation (Advanced)
|
|
253
|
+
|
|
254
|
+
pgmonkey v3.4.0 adds opt-in support for resolving environment variables and secret file references inside YAML configuration files. This lets you keep your config files free of hardcoded credentials while staying compatible with standard deployment workflows (12-factor env vars, Docker, Kubernetes).
|
|
255
|
+
|
|
256
|
+
**Interpolation is disabled by default.** If you do not enable it, pgmonkey treats every YAML value as a literal string - exactly as it always has.
|
|
257
|
+
|
|
258
|
+
### Standard YAML (Default)
|
|
259
|
+
|
|
260
|
+
The standard approach is to write literal values directly in the config file. This is the simplest way to get started and requires no extra flags or API parameters:
|
|
261
|
+
|
|
262
|
+
```yaml
|
|
263
|
+
connection_type: 'normal'
|
|
264
|
+
|
|
265
|
+
connection_settings:
|
|
266
|
+
user: 'postgres'
|
|
267
|
+
password: 'my_password'
|
|
268
|
+
host: 'localhost'
|
|
269
|
+
port: '5432'
|
|
270
|
+
dbname: 'mydatabase'
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
```python
|
|
274
|
+
from pgmonkey import PGConnectionManager
|
|
275
|
+
|
|
276
|
+
manager = PGConnectionManager()
|
|
277
|
+
conn = manager.get_database_connection('config.yaml')
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
This continues to work exactly as before. No changes needed for existing configs.
|
|
281
|
+
|
|
282
|
+
### Inline Syntax: ${VAR}
|
|
283
|
+
|
|
284
|
+
When interpolation is enabled, you can reference environment variables using `${VAR}` syntax. Use `${VAR:-default}` to provide a fallback value when the variable is not set:
|
|
285
|
+
|
|
286
|
+
```yaml
|
|
287
|
+
connection_type: 'normal'
|
|
288
|
+
|
|
289
|
+
connection_settings:
|
|
290
|
+
user: '${PGUSER:-postgres}'
|
|
291
|
+
password: '${PGPASSWORD}'
|
|
292
|
+
host: '${PGHOST:-localhost}'
|
|
293
|
+
port: '${PGPORT:-5432}'
|
|
294
|
+
dbname: '${PGDATABASE:-mydb}'
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
Rules:
|
|
298
|
+
- If a referenced variable is **not set** and **no default** is provided, pgmonkey raises `EnvInterpolationError` with a clear message naming the variable and the config key.
|
|
299
|
+
- Multiple references can appear in a single value: `'${PGHOST}:${PGPORT}'`
|
|
300
|
+
- Non-string values (integers, booleans) pass through unchanged.
|
|
301
|
+
|
|
302
|
+
### Structured Syntax: from_env / from_file
|
|
303
|
+
|
|
304
|
+
For secrets, pgmonkey supports a structured YAML form that makes the intent unambiguous:
|
|
305
|
+
|
|
306
|
+
**Read from an environment variable:**
|
|
307
|
+
|
|
308
|
+
```yaml
|
|
309
|
+
connection_settings:
|
|
310
|
+
password:
|
|
311
|
+
from_env: PGMONKEY_DB_PASSWORD
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
**Read from a file (Kubernetes Secret-style):**
|
|
315
|
+
|
|
316
|
+
```yaml
|
|
317
|
+
connection_settings:
|
|
318
|
+
password:
|
|
319
|
+
from_file: /var/run/secrets/db/password
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Rules:
|
|
323
|
+
- `from_env` reads the named environment variable. If it is not set, pgmonkey raises `EnvInterpolationError`.
|
|
324
|
+
- `from_file` reads the file contents and trims the trailing newline (matching Kubernetes Secret conventions). If the file does not exist or cannot be read, pgmonkey raises `EnvInterpolationError`.
|
|
325
|
+
- A structured reference must be a dict with exactly one key (`from_env` or `from_file`). Dicts with additional keys are treated as normal nested config.
|
|
326
|
+
|
|
327
|
+
### Sensitive Key Protection
|
|
328
|
+
|
|
329
|
+
By default, `${VAR:-default}` fallback values are **disallowed** for sensitive keys. This prevents accidentally shipping a config with a hardcoded fallback password that silently takes over when the env var is missing.
|
|
330
|
+
|
|
331
|
+
Sensitive keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`.
|
|
332
|
+
|
|
333
|
+
```yaml
|
|
334
|
+
# This will FAIL (password is a sensitive key, defaults not allowed):
|
|
335
|
+
password: '${PGPASSWORD:-fallback_password}'
|
|
336
|
+
|
|
337
|
+
# This is fine (host is not sensitive):
|
|
338
|
+
host: '${PGHOST:-localhost}'
|
|
339
|
+
```
|
|
340
|
+
|
|
341
|
+
To explicitly allow sensitive defaults (e.g. for local development), pass `allow_sensitive_defaults=True`:
|
|
342
|
+
|
|
343
|
+
```python
|
|
344
|
+
cfg = load_config('config.yaml', resolve_env=True, allow_sensitive_defaults=True)
|
|
345
|
+
```
|
|
346
|
+
|
|
347
|
+
### Python API
|
|
348
|
+
|
|
349
|
+
Use `load_config()` for the simplest path to a resolved config dictionary:
|
|
350
|
+
|
|
351
|
+
```python
|
|
352
|
+
from pgmonkey import load_config
|
|
353
|
+
|
|
354
|
+
# Load without interpolation (default - same as always)
|
|
355
|
+
cfg = load_config('config.yaml')
|
|
356
|
+
|
|
357
|
+
# Load with env interpolation enabled
|
|
358
|
+
cfg = load_config('config.yaml', resolve_env=True)
|
|
359
|
+
|
|
360
|
+
# Strict mode: fail on missing vars
|
|
361
|
+
cfg = load_config('config.yaml', resolve_env=True, strict=True)
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
Or pass `resolve_env=True` directly to the connection manager:
|
|
365
|
+
|
|
366
|
+
```python
|
|
367
|
+
from pgmonkey import PGConnectionManager
|
|
368
|
+
|
|
369
|
+
manager = PGConnectionManager()
|
|
370
|
+
conn = manager.get_database_connection('config.yaml', resolve_env=True)
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Both `get_database_connection()` and `get_database_connection_from_dict()` accept the `resolve_env` parameter.
|
|
374
|
+
|
|
375
|
+
### CLI Usage
|
|
376
|
+
|
|
377
|
+
Add `--resolve-env` to any `pgconfig test` or `pgconfig generate-code` command:
|
|
378
|
+
|
|
379
|
+
```bash
|
|
380
|
+
# Test a connection with env vars resolved
|
|
381
|
+
pgmonkey pgconfig test --connconfig config.yaml --resolve-env
|
|
382
|
+
|
|
383
|
+
# Generate code (--resolve-env accepted for consistency)
|
|
384
|
+
pgmonkey pgconfig generate-code --connconfig config.yaml --resolve-env
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
Without `--resolve-env`, the CLI works exactly as before - `${VAR}` patterns are treated as literal strings.
|
|
388
|
+
|
|
389
|
+
### Redacting Secrets for Logging
|
|
390
|
+
|
|
391
|
+
pgmonkey includes a `redact_config()` utility that replaces sensitive values with `***REDACTED***`, safe for logging or printing:
|
|
392
|
+
|
|
393
|
+
```python
|
|
394
|
+
from pgmonkey import load_config
|
|
395
|
+
from pgmonkey.common.utils.redaction import redact_config
|
|
396
|
+
|
|
397
|
+
cfg = load_config('config.yaml', resolve_env=True)
|
|
398
|
+
print(redact_config(cfg))
|
|
399
|
+
# {'connection_settings': {'password': '***REDACTED***', 'host': 'db.prod.com', ...}}
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
Redacted keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`. Empty strings and `None` values are left as-is (nothing to leak).
|
|
403
|
+
|
|
404
|
+
### Deployment Patterns
|
|
405
|
+
|
|
406
|
+
**Local development** - set env vars in your shell or a `.env` file (managed by your own tooling):
|
|
407
|
+
|
|
408
|
+
```bash
|
|
409
|
+
export PGPASSWORD=dev_password
|
|
410
|
+
export PGHOST=localhost
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
**Docker / containers** - pass env vars via `docker run -e` or `docker-compose.yml`:
|
|
414
|
+
|
|
415
|
+
```yaml
|
|
416
|
+
# docker-compose.yml
|
|
417
|
+
services:
|
|
418
|
+
app:
|
|
419
|
+
environment:
|
|
420
|
+
PGPASSWORD: ${DB_PASSWORD}
|
|
421
|
+
PGHOST: db
|
|
422
|
+
```
|
|
423
|
+
|
|
424
|
+
**Kubernetes** - mount secrets as files and use `from_file`:
|
|
425
|
+
|
|
426
|
+
```yaml
|
|
427
|
+
# pgmonkey config
|
|
428
|
+
connection_settings:
|
|
429
|
+
password:
|
|
430
|
+
from_file: /var/run/secrets/db/password
|
|
431
|
+
host: '${PGHOST:-db-service}'
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
```yaml
|
|
435
|
+
# k8s pod spec
|
|
436
|
+
volumes:
|
|
437
|
+
- name: db-secret
|
|
438
|
+
secret:
|
|
439
|
+
secretName: db-credentials
|
|
440
|
+
containers:
|
|
441
|
+
- volumeMounts:
|
|
442
|
+
- name: db-secret
|
|
443
|
+
mountPath: /var/run/secrets/db
|
|
444
|
+
readOnly: true
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
**Cloud secret managers** (AWS Secrets Manager, GCP Secret Manager, HashiCorp Vault) - resolve secrets in your deployment pipeline and set them as env vars. pgmonkey does not integrate with cloud vaults directly.
|
|
448
|
+
|
|
243
449
|
## Using the CLI
|
|
244
450
|
|
|
245
451
|
pgmonkey provides a command-line interface for managing configurations and connections.
|
|
@@ -27,6 +27,9 @@ def cli_pgconfig_subparser(subparsers):
|
|
|
27
27
|
help='Path to the configuration file you want to test.')
|
|
28
28
|
test_parser.add_argument('--connection-type', choices=CONNECTION_TYPE_CHOICES, default=None,
|
|
29
29
|
help='Connection type to test. Overrides the value in the config file.')
|
|
30
|
+
test_parser.add_argument('--resolve-env', action='store_true', default=False,
|
|
31
|
+
help='Resolve ${VAR} environment variable references and '
|
|
32
|
+
'from_env/from_file secret references in the config file.')
|
|
30
33
|
test_parser.set_defaults(func=pgconfig_test_handler, pgconfig_manager=pgconfig_manager)
|
|
31
34
|
|
|
32
35
|
# The "generate-code" subcommand
|
|
@@ -40,6 +43,9 @@ def cli_pgconfig_subparser(subparsers):
|
|
|
40
43
|
help='Target library for generated code. '
|
|
41
44
|
'"pgmonkey" (default) generates code using pgmonkey. '
|
|
42
45
|
'"psycopg" generates code using psycopg/psycopg_pool directly.')
|
|
46
|
+
code_parser.add_argument('--resolve-env', action='store_true', default=False,
|
|
47
|
+
help='Resolve ${VAR} environment variable references and '
|
|
48
|
+
'from_env/from_file secret references in the config file.')
|
|
43
49
|
code_parser.set_defaults(func=pgconfig_generate_code_handler, pgcodegen_manager=pgcodegen_manager)
|
|
44
50
|
|
|
45
51
|
|
|
@@ -56,11 +62,15 @@ def pgconfig_create_handler(args):
|
|
|
56
62
|
def pgconfig_test_handler(args):
|
|
57
63
|
pgconfig_manager = args.pgconfig_manager
|
|
58
64
|
connection_type = getattr(args, 'connection_type', None)
|
|
59
|
-
|
|
65
|
+
resolve_env = getattr(args, 'resolve_env', False)
|
|
66
|
+
pgconfig_manager.test_connection(args.filepath, connection_type, resolve_env=resolve_env)
|
|
60
67
|
|
|
61
68
|
|
|
62
69
|
def pgconfig_generate_code_handler(args):
|
|
63
70
|
pgcodegen_manager = args.pgcodegen_manager
|
|
64
71
|
connection_type = getattr(args, 'connection_type', None)
|
|
65
72
|
library = getattr(args, 'library', 'pgmonkey')
|
|
66
|
-
|
|
73
|
+
resolve_env = getattr(args, 'resolve_env', False)
|
|
74
|
+
pgcodegen_manager.generate_connection_code(
|
|
75
|
+
args.filepath, connection_type, library=library, resolve_env=resolve_env,
|
|
76
|
+
)
|
|
@@ -63,3 +63,21 @@ async_pool_settings:
|
|
|
63
63
|
# max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
|
|
64
64
|
# reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
|
|
65
65
|
# num_workers: 3 # Number of background workers maintaining pool connections
|
|
66
|
+
|
|
67
|
+
# -- Advanced: environment variable interpolation (opt-in) --
|
|
68
|
+
# By default, pgmonkey uses the literal values above as-is.
|
|
69
|
+
# If you need to inject secrets at runtime instead of hardcoding them,
|
|
70
|
+
# enable interpolation with resolve_env=True in Python or --resolve-env on the CLI.
|
|
71
|
+
#
|
|
72
|
+
# Inline syntax:
|
|
73
|
+
# host: '${PGHOST:-localhost}' # uses env var, falls back to 'localhost'
|
|
74
|
+
# password: '${PGPASSWORD}' # required - error if not set
|
|
75
|
+
#
|
|
76
|
+
# Structured syntax (recommended for secrets):
|
|
77
|
+
# password:
|
|
78
|
+
# from_env: PGMONKEY_DB_PASSWORD # reads env var
|
|
79
|
+
# password:
|
|
80
|
+
# from_file: /var/run/secrets/db/pass # reads file, trims trailing newline
|
|
81
|
+
#
|
|
82
|
+
# Sensitive keys (password, sslkey, sslcert, sslrootcert) do not allow
|
|
83
|
+
# defaults in ${VAR:-default} by default, to prevent accidental fallbacks.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
|
|
3
|
+
import yaml
|
|
4
|
+
|
|
5
|
+
from pgmonkey.common.utils.envutils import resolve_env_vars
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_config(file_path, resolve_env=False, strict=False,
|
|
9
|
+
allow_sensitive_defaults=False):
|
|
10
|
+
"""Load and optionally interpolate a pgmonkey YAML configuration file.
|
|
11
|
+
|
|
12
|
+
This is the recommended entry point for programmatic config loading.
|
|
13
|
+
|
|
14
|
+
Parameters
|
|
15
|
+
----------
|
|
16
|
+
file_path : str
|
|
17
|
+
Path to the YAML configuration file.
|
|
18
|
+
resolve_env : bool
|
|
19
|
+
If True, ``${VAR}`` / ``${VAR:-default}`` patterns and
|
|
20
|
+
``from_env`` / ``from_file`` structured references are resolved.
|
|
21
|
+
strict : bool
|
|
22
|
+
If True (and *resolve_env* is True), missing env vars with no
|
|
23
|
+
default raise immediately. Passed through to
|
|
24
|
+
:func:`resolve_env_vars`.
|
|
25
|
+
allow_sensitive_defaults : bool
|
|
26
|
+
If True, ``${VAR:-default}`` is permitted even for sensitive keys
|
|
27
|
+
like ``password``. Default is False for safety.
|
|
28
|
+
|
|
29
|
+
Returns
|
|
30
|
+
-------
|
|
31
|
+
dict
|
|
32
|
+
The parsed (and optionally resolved) configuration dictionary.
|
|
33
|
+
"""
|
|
34
|
+
with open(file_path, 'r') as f:
|
|
35
|
+
config = yaml.safe_load(f)
|
|
36
|
+
|
|
37
|
+
config = normalize_config(config)
|
|
38
|
+
|
|
39
|
+
if resolve_env:
|
|
40
|
+
config = resolve_env_vars(
|
|
41
|
+
config,
|
|
42
|
+
strict=strict,
|
|
43
|
+
allow_sensitive_defaults=allow_sensitive_defaults,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return config
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def normalize_config(config_data):
|
|
50
|
+
"""Normalize a pgmonkey configuration dictionary.
|
|
51
|
+
|
|
52
|
+
In pgmonkey v3.0.0, the top-level 'postgresql:' wrapper key was removed.
|
|
53
|
+
This function detects the old format and unwraps it with a deprecation warning.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
config_data: Configuration dictionary (old or new format).
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
The normalized configuration dictionary (without the 'postgresql:' wrapper).
|
|
60
|
+
"""
|
|
61
|
+
if isinstance(config_data, dict) and 'postgresql' in config_data:
|
|
62
|
+
warnings.warn(
|
|
63
|
+
"The top-level 'postgresql:' key in pgmonkey config files is deprecated "
|
|
64
|
+
"since v3.0.0 and will be removed in a future version. "
|
|
65
|
+
"Remove the 'postgresql:' wrapper and dedent all settings one level. "
|
|
66
|
+
"See https://pgmonkey.net/reference.html for the new format.",
|
|
67
|
+
DeprecationWarning,
|
|
68
|
+
stacklevel=3,
|
|
69
|
+
)
|
|
70
|
+
return config_data['postgresql']
|
|
71
|
+
return config_data
|