pgmonkey 3.2.0__tar.gz → 3.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. {pgmonkey-3.2.0/src/pgmonkey.egg-info → pgmonkey-3.4.0}/PKG-INFO +213 -7
  2. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/README.md +212 -6
  3. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/pyproject.toml +1 -1
  4. pgmonkey-3.4.0/src/pgmonkey/__init__.py +3 -0
  5. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_pgconfig_subparser.py +12 -2
  6. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/templates/postgres.yaml +18 -0
  7. pgmonkey-3.4.0/src/pgmonkey/common/utils/configutils.py +71 -0
  8. pgmonkey-3.4.0/src/pgmonkey/common/utils/envutils.py +212 -0
  9. pgmonkey-3.4.0/src/pgmonkey/common/utils/redaction.py +49 -0
  10. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgcodegen_manager.py +4 -1
  11. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgconfig_manager.py +5 -2
  12. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgconnection_manager.py +15 -2
  13. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgexport_manager.py +1 -2
  14. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pgimport_manager.py +1 -2
  15. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_csv_data_exporter.py +9 -0
  16. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_csv_data_importer.py +118 -0
  17. pgmonkey-3.4.0/src/pgmonkey/tests/unit/test_env_interpolation.py +490 -0
  18. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/csv_data_exporter.py +1 -1
  19. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/csv_data_importer.py +9 -2
  20. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/database_connection_tester.py +19 -7
  21. {pgmonkey-3.2.0 → pgmonkey-3.4.0/src/pgmonkey.egg-info}/PKG-INFO +213 -7
  22. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/SOURCES.txt +3 -0
  23. pgmonkey-3.2.0/src/pgmonkey/__init__.py +0 -1
  24. pgmonkey-3.2.0/src/pgmonkey/common/utils/configutils.py +0 -26
  25. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/LICENSE +0 -0
  26. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/NOTICE +0 -0
  27. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/setup.cfg +0 -0
  28. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/__init__.py +0 -0
  29. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/__init__.py +0 -0
  30. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli.py +0 -0
  31. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_export_subparser.py +0 -0
  32. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_import_subparser.py +0 -0
  33. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_pg_server_config_subparser.py +0 -0
  34. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_settings_subparser.py +0 -0
  35. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/cli/cli_toplevel_parser.py +0 -0
  36. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/__init__.py +0 -0
  37. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/config/__init__.py +0 -0
  38. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/exceptions.py +0 -0
  39. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/utils/__init__.py +0 -0
  40. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/common/utils/pathutils.py +0 -0
  41. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/__init__.py +0 -0
  42. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/base.py +0 -0
  43. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/__init__.py +0 -0
  44. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/async_connection.py +0 -0
  45. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/async_pool_connection.py +0 -0
  46. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/base_connection.py +0 -0
  47. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/normal_connection.py +0 -0
  48. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/pool_connection.py +0 -0
  49. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/connections/postgres/postgres_connection_factory.py +0 -0
  50. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/__init__.py +0 -0
  51. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/pg_server_config_manager.py +0 -0
  52. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/settings_manager.py +0 -0
  53. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/managers/toplevel_manager.py +0 -0
  54. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/serversettings/postgres_server_config_generator.py +0 -0
  55. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/serversettings/postgres_server_settings_inspector.py +0 -0
  56. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/settings/app_settings.yaml +0 -0
  57. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/__init__.py +0 -0
  58. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/conftest.py +0 -0
  59. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/integration/__init__.py +0 -0
  60. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/integration/test_pgconnection_manager_integration.py +0 -0
  61. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/__init__.py +0 -0
  62. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_async_connection.py +0 -0
  63. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_async_pool_connection.py +0 -0
  64. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_base_connection.py +0 -0
  65. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_code_generator.py +0 -0
  66. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_config_manager.py +0 -0
  67. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_connection_caching.py +0 -0
  68. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_connection_factory.py +0 -0
  69. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_normal_connection.py +0 -0
  70. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_path_utils.py +0 -0
  71. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_pgconnection_manager.py +0 -0
  72. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_pool_connection.py +0 -0
  73. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_server_config_generator.py +0 -0
  74. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_server_settings_inspector.py +0 -0
  75. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tests/unit/test_settings_manager.py +0 -0
  76. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/__init__.py +0 -0
  77. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey/tools/connection_code_generator.py +0 -0
  78. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/dependency_links.txt +0 -0
  79. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/entry_points.txt +0 -0
  80. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/requires.txt +0 -0
  81. {pgmonkey-3.2.0 → pgmonkey-3.4.0}/src/pgmonkey.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pgmonkey
3
- Version: 3.2.0
3
+ Version: 3.4.0
4
4
  Summary: A tool to assist with postgresql database connections
5
5
  Author-email: Good Boy <pythonic@rexbytes.com>
6
6
  License: MIT
@@ -47,24 +47,33 @@ Dynamic: license-file
47
47
  - [Password-Based Authentication](#password-based-authentication)
48
48
  - [SSL/TLS Encryption](#ssltls-encryption)
49
49
  - [Certificate-Based Authentication](#certificate-based-authentication)
50
- 5. [Using the CLI](#using-the-cli)
50
+ 5. [Environment Variable Interpolation (Advanced)](#environment-variable-interpolation-advanced)
51
+ - [Standard YAML (Default)](#standard-yaml-default)
52
+ - [Inline Syntax: ${VAR}](#inline-syntax-var)
53
+ - [Structured Syntax: from_env / from_file](#structured-syntax-from_env--from_file)
54
+ - [Sensitive Key Protection](#sensitive-key-protection)
55
+ - [Python API](#python-api)
56
+ - [CLI Usage](#cli-usage)
57
+ - [Redacting Secrets for Logging](#redacting-secrets-for-logging)
58
+ - [Deployment Patterns](#deployment-patterns)
59
+ 6. [Using the CLI](#using-the-cli)
51
60
  - [Creating a Configuration Template](#creating-a-configuration-template)
52
61
  - [Testing a Connection](#testing-a-connection)
53
62
  - [Generating Python Code](#generating-python-code)
54
63
  - [Server Configuration Recommendations](#server-configuration-recommendations)
55
64
  - [Auditing Live Server Settings](#auditing-live-server-settings)
56
65
  - [Importing and Exporting Data](#importing-and-exporting-data)
57
- 6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
66
+ 7. [Using pgmonkey in Python](#using-pgmonkey-in-python)
58
67
  - [Normal (Synchronous) Connection](#normal-synchronous-connection)
59
68
  - [Pooled Connection](#pooled-connection)
60
69
  - [Async Connection](#async-connection)
61
70
  - [Async Pooled Connection](#async-pooled-connection)
62
71
  - [Using the Config File Default](#using-the-config-file-default)
63
72
  - [Transactions, Commit, and Rollback](#transactions-commit-and-rollback)
64
- 7. [Best Practice Recipes](#best-practice-recipes)
65
- 8. [Testing All Connection Types](#testing-all-connection-types)
66
- 9. [Testing Pool Capacity](#testing-pool-capacity)
67
- 10. [Running the Test Suite](#running-the-test-suite)
73
+ 8. [Best Practice Recipes](#best-practice-recipes)
74
+ 9. [Testing All Connection Types](#testing-all-connection-types)
75
+ 10. [Testing Pool Capacity](#testing-pool-capacity)
76
+ 11. [Running the Test Suite](#running-the-test-suite)
68
77
 
69
78
  ## Installation
70
79
 
@@ -272,6 +281,203 @@ connection_settings:
272
281
  sslrootcert: '/path/to/ca.crt'
273
282
  ```
274
283
 
284
+ ## Environment Variable Interpolation (Advanced)
285
+
286
+ pgmonkey v3.4.0 adds opt-in support for resolving environment variables and secret file references inside YAML configuration files. This lets you keep your config files free of hardcoded credentials while staying compatible with standard deployment workflows (12-factor env vars, Docker, Kubernetes).
287
+
288
+ **Interpolation is disabled by default.** If you do not enable it, pgmonkey treats every YAML value as a literal string - exactly as it always has.
289
+
290
+ ### Standard YAML (Default)
291
+
292
+ The standard approach is to write literal values directly in the config file. This is the simplest way to get started and requires no extra flags or API parameters:
293
+
294
+ ```yaml
295
+ connection_type: 'normal'
296
+
297
+ connection_settings:
298
+ user: 'postgres'
299
+ password: 'my_password'
300
+ host: 'localhost'
301
+ port: '5432'
302
+ dbname: 'mydatabase'
303
+ ```
304
+
305
+ ```python
306
+ from pgmonkey import PGConnectionManager
307
+
308
+ manager = PGConnectionManager()
309
+ conn = manager.get_database_connection('config.yaml')
310
+ ```
311
+
312
+ This continues to work exactly as before. No changes needed for existing configs.
313
+
314
+ ### Inline Syntax: ${VAR}
315
+
316
+ When interpolation is enabled, you can reference environment variables using `${VAR}` syntax. Use `${VAR:-default}` to provide a fallback value when the variable is not set:
317
+
318
+ ```yaml
319
+ connection_type: 'normal'
320
+
321
+ connection_settings:
322
+ user: '${PGUSER:-postgres}'
323
+ password: '${PGPASSWORD}'
324
+ host: '${PGHOST:-localhost}'
325
+ port: '${PGPORT:-5432}'
326
+ dbname: '${PGDATABASE:-mydb}'
327
+ ```
328
+
329
+ Rules:
330
+ - If a referenced variable is **not set** and **no default** is provided, pgmonkey raises `EnvInterpolationError` with a clear message naming the variable and the config key.
331
+ - Multiple references can appear in a single value: `'${PGHOST}:${PGPORT}'`
332
+ - Non-string values (integers, booleans) pass through unchanged.
333
+
334
+ ### Structured Syntax: from_env / from_file
335
+
336
+ For secrets, pgmonkey supports a structured YAML form that makes the intent unambiguous:
337
+
338
+ **Read from an environment variable:**
339
+
340
+ ```yaml
341
+ connection_settings:
342
+ password:
343
+ from_env: PGMONKEY_DB_PASSWORD
344
+ ```
345
+
346
+ **Read from a file (Kubernetes Secret-style):**
347
+
348
+ ```yaml
349
+ connection_settings:
350
+ password:
351
+ from_file: /var/run/secrets/db/password
352
+ ```
353
+
354
+ Rules:
355
+ - `from_env` reads the named environment variable. If it is not set, pgmonkey raises `EnvInterpolationError`.
356
+ - `from_file` reads the file contents and trims the trailing newline (matching Kubernetes Secret conventions). If the file does not exist or cannot be read, pgmonkey raises `EnvInterpolationError`.
357
+ - A structured reference must be a dict with exactly one key (`from_env` or `from_file`). Dicts with additional keys are treated as normal nested config.
358
+
359
+ ### Sensitive Key Protection
360
+
361
+ By default, `${VAR:-default}` fallback values are **disallowed** for sensitive keys. This prevents accidentally shipping a config with a hardcoded fallback password that silently takes over when the env var is missing.
362
+
363
+ Sensitive keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`.
364
+
365
+ ```yaml
366
+ # This will FAIL (password is a sensitive key, defaults not allowed):
367
+ password: '${PGPASSWORD:-fallback_password}'
368
+
369
+ # This is fine (host is not sensitive):
370
+ host: '${PGHOST:-localhost}'
371
+ ```
372
+
373
+ To explicitly allow sensitive defaults (e.g. for local development), pass `allow_sensitive_defaults=True`:
374
+
375
+ ```python
376
+ cfg = load_config('config.yaml', resolve_env=True, allow_sensitive_defaults=True)
377
+ ```
378
+
379
+ ### Python API
380
+
381
+ Use `load_config()` for the simplest path to a resolved config dictionary:
382
+
383
+ ```python
384
+ from pgmonkey import load_config
385
+
386
+ # Load without interpolation (default - same as always)
387
+ cfg = load_config('config.yaml')
388
+
389
+ # Load with env interpolation enabled
390
+ cfg = load_config('config.yaml', resolve_env=True)
391
+
392
+ # Strict mode: fail on missing vars
393
+ cfg = load_config('config.yaml', resolve_env=True, strict=True)
394
+ ```
395
+
396
+ Or pass `resolve_env=True` directly to the connection manager:
397
+
398
+ ```python
399
+ from pgmonkey import PGConnectionManager
400
+
401
+ manager = PGConnectionManager()
402
+ conn = manager.get_database_connection('config.yaml', resolve_env=True)
403
+ ```
404
+
405
+ Both `get_database_connection()` and `get_database_connection_from_dict()` accept the `resolve_env` parameter.
406
+
407
+ ### CLI Usage
408
+
409
+ Add `--resolve-env` to any `pgconfig test` or `pgconfig generate-code` command:
410
+
411
+ ```bash
412
+ # Test a connection with env vars resolved
413
+ pgmonkey pgconfig test --connconfig config.yaml --resolve-env
414
+
415
+ # Generate code (--resolve-env accepted for consistency)
416
+ pgmonkey pgconfig generate-code --connconfig config.yaml --resolve-env
417
+ ```
418
+
419
+ Without `--resolve-env`, the CLI works exactly as before - `${VAR}` patterns are treated as literal strings.
420
+
421
+ ### Redacting Secrets for Logging
422
+
423
+ pgmonkey includes a `redact_config()` utility that replaces sensitive values with `***REDACTED***`, safe for logging or printing:
424
+
425
+ ```python
426
+ from pgmonkey import load_config
427
+ from pgmonkey.common.utils.redaction import redact_config
428
+
429
+ cfg = load_config('config.yaml', resolve_env=True)
430
+ print(redact_config(cfg))
431
+ # {'connection_settings': {'password': '***REDACTED***', 'host': 'db.prod.com', ...}}
432
+ ```
433
+
434
+ Redacted keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`. Empty strings and `None` values are left as-is (nothing to leak).
435
+
436
+ ### Deployment Patterns
437
+
438
+ **Local development** - set env vars in your shell or a `.env` file (managed by your own tooling):
439
+
440
+ ```bash
441
+ export PGPASSWORD=dev_password
442
+ export PGHOST=localhost
443
+ ```
444
+
445
+ **Docker / containers** - pass env vars via `docker run -e` or `docker-compose.yml`:
446
+
447
+ ```yaml
448
+ # docker-compose.yml
449
+ services:
450
+ app:
451
+ environment:
452
+ PGPASSWORD: ${DB_PASSWORD}
453
+ PGHOST: db
454
+ ```
455
+
456
+ **Kubernetes** - mount secrets as files and use `from_file`:
457
+
458
+ ```yaml
459
+ # pgmonkey config
460
+ connection_settings:
461
+ password:
462
+ from_file: /var/run/secrets/db/password
463
+ host: '${PGHOST:-db-service}'
464
+ ```
465
+
466
+ ```yaml
467
+ # k8s pod spec
468
+ volumes:
469
+ - name: db-secret
470
+ secret:
471
+ secretName: db-credentials
472
+ containers:
473
+ - volumeMounts:
474
+ - name: db-secret
475
+ mountPath: /var/run/secrets/db
476
+ readOnly: true
477
+ ```
478
+
479
+ **Cloud secret managers** (AWS Secrets Manager, GCP Secret Manager, HashiCorp Vault) - resolve secrets in your deployment pipeline and set them as env vars. pgmonkey does not integrate with cloud vaults directly.
480
+
275
481
  ## Using the CLI
276
482
 
277
483
  pgmonkey provides a command-line interface for managing configurations and connections.
@@ -15,24 +15,33 @@
15
15
  - [Password-Based Authentication](#password-based-authentication)
16
16
  - [SSL/TLS Encryption](#ssltls-encryption)
17
17
  - [Certificate-Based Authentication](#certificate-based-authentication)
18
- 5. [Using the CLI](#using-the-cli)
18
+ 5. [Environment Variable Interpolation (Advanced)](#environment-variable-interpolation-advanced)
19
+ - [Standard YAML (Default)](#standard-yaml-default)
20
+ - [Inline Syntax: ${VAR}](#inline-syntax-var)
21
+ - [Structured Syntax: from_env / from_file](#structured-syntax-from_env--from_file)
22
+ - [Sensitive Key Protection](#sensitive-key-protection)
23
+ - [Python API](#python-api)
24
+ - [CLI Usage](#cli-usage)
25
+ - [Redacting Secrets for Logging](#redacting-secrets-for-logging)
26
+ - [Deployment Patterns](#deployment-patterns)
27
+ 6. [Using the CLI](#using-the-cli)
19
28
  - [Creating a Configuration Template](#creating-a-configuration-template)
20
29
  - [Testing a Connection](#testing-a-connection)
21
30
  - [Generating Python Code](#generating-python-code)
22
31
  - [Server Configuration Recommendations](#server-configuration-recommendations)
23
32
  - [Auditing Live Server Settings](#auditing-live-server-settings)
24
33
  - [Importing and Exporting Data](#importing-and-exporting-data)
25
- 6. [Using pgmonkey in Python](#using-pgmonkey-in-python)
34
+ 7. [Using pgmonkey in Python](#using-pgmonkey-in-python)
26
35
  - [Normal (Synchronous) Connection](#normal-synchronous-connection)
27
36
  - [Pooled Connection](#pooled-connection)
28
37
  - [Async Connection](#async-connection)
29
38
  - [Async Pooled Connection](#async-pooled-connection)
30
39
  - [Using the Config File Default](#using-the-config-file-default)
31
40
  - [Transactions, Commit, and Rollback](#transactions-commit-and-rollback)
32
- 7. [Best Practice Recipes](#best-practice-recipes)
33
- 8. [Testing All Connection Types](#testing-all-connection-types)
34
- 9. [Testing Pool Capacity](#testing-pool-capacity)
35
- 10. [Running the Test Suite](#running-the-test-suite)
41
+ 8. [Best Practice Recipes](#best-practice-recipes)
42
+ 9. [Testing All Connection Types](#testing-all-connection-types)
43
+ 10. [Testing Pool Capacity](#testing-pool-capacity)
44
+ 11. [Running the Test Suite](#running-the-test-suite)
36
45
 
37
46
  ## Installation
38
47
 
@@ -240,6 +249,203 @@ connection_settings:
240
249
  sslrootcert: '/path/to/ca.crt'
241
250
  ```
242
251
 
252
+ ## Environment Variable Interpolation (Advanced)
253
+
254
+ pgmonkey v3.4.0 adds opt-in support for resolving environment variables and secret file references inside YAML configuration files. This lets you keep your config files free of hardcoded credentials while staying compatible with standard deployment workflows (12-factor env vars, Docker, Kubernetes).
255
+
256
+ **Interpolation is disabled by default.** If you do not enable it, pgmonkey treats every YAML value as a literal string - exactly as it always has.
257
+
258
+ ### Standard YAML (Default)
259
+
260
+ The standard approach is to write literal values directly in the config file. This is the simplest way to get started and requires no extra flags or API parameters:
261
+
262
+ ```yaml
263
+ connection_type: 'normal'
264
+
265
+ connection_settings:
266
+ user: 'postgres'
267
+ password: 'my_password'
268
+ host: 'localhost'
269
+ port: '5432'
270
+ dbname: 'mydatabase'
271
+ ```
272
+
273
+ ```python
274
+ from pgmonkey import PGConnectionManager
275
+
276
+ manager = PGConnectionManager()
277
+ conn = manager.get_database_connection('config.yaml')
278
+ ```
279
+
280
+ This continues to work exactly as before. No changes needed for existing configs.
281
+
282
+ ### Inline Syntax: ${VAR}
283
+
284
+ When interpolation is enabled, you can reference environment variables using `${VAR}` syntax. Use `${VAR:-default}` to provide a fallback value when the variable is not set:
285
+
286
+ ```yaml
287
+ connection_type: 'normal'
288
+
289
+ connection_settings:
290
+ user: '${PGUSER:-postgres}'
291
+ password: '${PGPASSWORD}'
292
+ host: '${PGHOST:-localhost}'
293
+ port: '${PGPORT:-5432}'
294
+ dbname: '${PGDATABASE:-mydb}'
295
+ ```
296
+
297
+ Rules:
298
+ - If a referenced variable is **not set** and **no default** is provided, pgmonkey raises `EnvInterpolationError` with a clear message naming the variable and the config key.
299
+ - Multiple references can appear in a single value: `'${PGHOST}:${PGPORT}'`
300
+ - Non-string values (integers, booleans) pass through unchanged.
301
+
302
+ ### Structured Syntax: from_env / from_file
303
+
304
+ For secrets, pgmonkey supports a structured YAML form that makes the intent unambiguous:
305
+
306
+ **Read from an environment variable:**
307
+
308
+ ```yaml
309
+ connection_settings:
310
+ password:
311
+ from_env: PGMONKEY_DB_PASSWORD
312
+ ```
313
+
314
+ **Read from a file (Kubernetes Secret-style):**
315
+
316
+ ```yaml
317
+ connection_settings:
318
+ password:
319
+ from_file: /var/run/secrets/db/password
320
+ ```
321
+
322
+ Rules:
323
+ - `from_env` reads the named environment variable. If it is not set, pgmonkey raises `EnvInterpolationError`.
324
+ - `from_file` reads the file contents and trims the trailing newline (matching Kubernetes Secret conventions). If the file does not exist or cannot be read, pgmonkey raises `EnvInterpolationError`.
325
+ - A structured reference must be a dict with exactly one key (`from_env` or `from_file`). Dicts with additional keys are treated as normal nested config.
326
+
327
+ ### Sensitive Key Protection
328
+
329
+ By default, `${VAR:-default}` fallback values are **disallowed** for sensitive keys. This prevents accidentally shipping a config with a hardcoded fallback password that silently takes over when the env var is missing.
330
+
331
+ Sensitive keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`.
332
+
333
+ ```yaml
334
+ # This will FAIL (password is a sensitive key, defaults not allowed):
335
+ password: '${PGPASSWORD:-fallback_password}'
336
+
337
+ # This is fine (host is not sensitive):
338
+ host: '${PGHOST:-localhost}'
339
+ ```
340
+
341
+ To explicitly allow sensitive defaults (e.g. for local development), pass `allow_sensitive_defaults=True`:
342
+
343
+ ```python
344
+ cfg = load_config('config.yaml', resolve_env=True, allow_sensitive_defaults=True)
345
+ ```
346
+
347
+ ### Python API
348
+
349
+ Use `load_config()` for the simplest path to a resolved config dictionary:
350
+
351
+ ```python
352
+ from pgmonkey import load_config
353
+
354
+ # Load without interpolation (default - same as always)
355
+ cfg = load_config('config.yaml')
356
+
357
+ # Load with env interpolation enabled
358
+ cfg = load_config('config.yaml', resolve_env=True)
359
+
360
+ # Strict mode: fail on missing vars
361
+ cfg = load_config('config.yaml', resolve_env=True, strict=True)
362
+ ```
363
+
364
+ Or pass `resolve_env=True` directly to the connection manager:
365
+
366
+ ```python
367
+ from pgmonkey import PGConnectionManager
368
+
369
+ manager = PGConnectionManager()
370
+ conn = manager.get_database_connection('config.yaml', resolve_env=True)
371
+ ```
372
+
373
+ Both `get_database_connection()` and `get_database_connection_from_dict()` accept the `resolve_env` parameter.
374
+
375
+ ### CLI Usage
376
+
377
+ Add `--resolve-env` to any `pgconfig test` or `pgconfig generate-code` command:
378
+
379
+ ```bash
380
+ # Test a connection with env vars resolved
381
+ pgmonkey pgconfig test --connconfig config.yaml --resolve-env
382
+
383
+ # Generate code (--resolve-env accepted for consistency)
384
+ pgmonkey pgconfig generate-code --connconfig config.yaml --resolve-env
385
+ ```
386
+
387
+ Without `--resolve-env`, the CLI works exactly as before - `${VAR}` patterns are treated as literal strings.
388
+
389
+ ### Redacting Secrets for Logging
390
+
391
+ pgmonkey includes a `redact_config()` utility that replaces sensitive values with `***REDACTED***`, safe for logging or printing:
392
+
393
+ ```python
394
+ from pgmonkey import load_config
395
+ from pgmonkey.common.utils.redaction import redact_config
396
+
397
+ cfg = load_config('config.yaml', resolve_env=True)
398
+ print(redact_config(cfg))
399
+ # {'connection_settings': {'password': '***REDACTED***', 'host': 'db.prod.com', ...}}
400
+ ```
401
+
402
+ Redacted keys: `password`, `sslkey`, `sslcert`, `sslrootcert`, and any key containing `token`, `secret`, or `credential`. Empty strings and `None` values are left as-is (nothing to leak).
403
+
404
+ ### Deployment Patterns
405
+
406
+ **Local development** - set env vars in your shell or a `.env` file (managed by your own tooling):
407
+
408
+ ```bash
409
+ export PGPASSWORD=dev_password
410
+ export PGHOST=localhost
411
+ ```
412
+
413
+ **Docker / containers** - pass env vars via `docker run -e` or `docker-compose.yml`:
414
+
415
+ ```yaml
416
+ # docker-compose.yml
417
+ services:
418
+ app:
419
+ environment:
420
+ PGPASSWORD: ${DB_PASSWORD}
421
+ PGHOST: db
422
+ ```
423
+
424
+ **Kubernetes** - mount secrets as files and use `from_file`:
425
+
426
+ ```yaml
427
+ # pgmonkey config
428
+ connection_settings:
429
+ password:
430
+ from_file: /var/run/secrets/db/password
431
+ host: '${PGHOST:-db-service}'
432
+ ```
433
+
434
+ ```yaml
435
+ # k8s pod spec
436
+ volumes:
437
+ - name: db-secret
438
+ secret:
439
+ secretName: db-credentials
440
+ containers:
441
+ - volumeMounts:
442
+ - name: db-secret
443
+ mountPath: /var/run/secrets/db
444
+ readOnly: true
445
+ ```
446
+
447
+ **Cloud secret managers** (AWS Secrets Manager, GCP Secret Manager, HashiCorp Vault) - resolve secrets in your deployment pipeline and set them as env vars. pgmonkey does not integrate with cloud vaults directly.
448
+
243
449
  ## Using the CLI
244
450
 
245
451
  pgmonkey provides a command-line interface for managing configurations and connections.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "pgmonkey"
7
- version = "3.2.0"
7
+ version = "3.4.0"
8
8
  authors = [
9
9
  { name="Good Boy", email="pythonic@rexbytes.com" },
10
10
  ]
@@ -0,0 +1,3 @@
1
+ from .managers.pgconnection_manager import PGConnectionManager
2
+ from .common.utils.configutils import load_config
3
+ from .common.utils.envutils import EnvInterpolationError
@@ -27,6 +27,9 @@ def cli_pgconfig_subparser(subparsers):
27
27
  help='Path to the configuration file you want to test.')
28
28
  test_parser.add_argument('--connection-type', choices=CONNECTION_TYPE_CHOICES, default=None,
29
29
  help='Connection type to test. Overrides the value in the config file.')
30
+ test_parser.add_argument('--resolve-env', action='store_true', default=False,
31
+ help='Resolve ${VAR} environment variable references and '
32
+ 'from_env/from_file secret references in the config file.')
30
33
  test_parser.set_defaults(func=pgconfig_test_handler, pgconfig_manager=pgconfig_manager)
31
34
 
32
35
  # The "generate-code" subcommand
@@ -40,6 +43,9 @@ def cli_pgconfig_subparser(subparsers):
40
43
  help='Target library for generated code. '
41
44
  '"pgmonkey" (default) generates code using pgmonkey. '
42
45
  '"psycopg" generates code using psycopg/psycopg_pool directly.')
46
+ code_parser.add_argument('--resolve-env', action='store_true', default=False,
47
+ help='Resolve ${VAR} environment variable references and '
48
+ 'from_env/from_file secret references in the config file.')
43
49
  code_parser.set_defaults(func=pgconfig_generate_code_handler, pgcodegen_manager=pgcodegen_manager)
44
50
 
45
51
 
@@ -56,11 +62,15 @@ def pgconfig_create_handler(args):
56
62
  def pgconfig_test_handler(args):
57
63
  pgconfig_manager = args.pgconfig_manager
58
64
  connection_type = getattr(args, 'connection_type', None)
59
- pgconfig_manager.test_connection(args.filepath, connection_type)
65
+ resolve_env = getattr(args, 'resolve_env', False)
66
+ pgconfig_manager.test_connection(args.filepath, connection_type, resolve_env=resolve_env)
60
67
 
61
68
 
62
69
  def pgconfig_generate_code_handler(args):
63
70
  pgcodegen_manager = args.pgcodegen_manager
64
71
  connection_type = getattr(args, 'connection_type', None)
65
72
  library = getattr(args, 'library', 'pgmonkey')
66
- pgcodegen_manager.generate_connection_code(args.filepath, connection_type, library=library)
73
+ resolve_env = getattr(args, 'resolve_env', False)
74
+ pgcodegen_manager.generate_connection_code(
75
+ args.filepath, connection_type, library=library, resolve_env=resolve_env,
76
+ )
@@ -63,3 +63,21 @@ async_pool_settings:
63
63
  # max_waiting: 0 # Max clients queued waiting for a connection (0 = unlimited)
64
64
  # reconnect_timeout: 300 # Seconds to keep trying to reconnect failed connections (0 = forever)
65
65
  # num_workers: 3 # Number of background workers maintaining pool connections
66
+
67
+ # -- Advanced: environment variable interpolation (opt-in) --
68
+ # By default, pgmonkey uses the literal values above as-is.
69
+ # If you need to inject secrets at runtime instead of hardcoding them,
70
+ # enable interpolation with resolve_env=True in Python or --resolve-env on the CLI.
71
+ #
72
+ # Inline syntax:
73
+ # host: '${PGHOST:-localhost}' # uses env var, falls back to 'localhost'
74
+ # password: '${PGPASSWORD}' # required - error if not set
75
+ #
76
+ # Structured syntax (recommended for secrets):
77
+ # password:
78
+ # from_env: PGMONKEY_DB_PASSWORD # reads env var
79
+ # password:
80
+ # from_file: /var/run/secrets/db/pass # reads file, trims trailing newline
81
+ #
82
+ # Sensitive keys (password, sslkey, sslcert, sslrootcert) do not allow
83
+ # defaults in ${VAR:-default} by default, to prevent accidental fallbacks.
@@ -0,0 +1,71 @@
1
+ import warnings
2
+
3
+ import yaml
4
+
5
+ from pgmonkey.common.utils.envutils import resolve_env_vars
6
+
7
+
8
+ def load_config(file_path, resolve_env=False, strict=False,
9
+ allow_sensitive_defaults=False):
10
+ """Load and optionally interpolate a pgmonkey YAML configuration file.
11
+
12
+ This is the recommended entry point for programmatic config loading.
13
+
14
+ Parameters
15
+ ----------
16
+ file_path : str
17
+ Path to the YAML configuration file.
18
+ resolve_env : bool
19
+ If True, ``${VAR}`` / ``${VAR:-default}`` patterns and
20
+ ``from_env`` / ``from_file`` structured references are resolved.
21
+ strict : bool
22
+ If True (and *resolve_env* is True), missing env vars with no
23
+ default raise immediately. Passed through to
24
+ :func:`resolve_env_vars`.
25
+ allow_sensitive_defaults : bool
26
+ If True, ``${VAR:-default}`` is permitted even for sensitive keys
27
+ like ``password``. Default is False for safety.
28
+
29
+ Returns
30
+ -------
31
+ dict
32
+ The parsed (and optionally resolved) configuration dictionary.
33
+ """
34
+ with open(file_path, 'r') as f:
35
+ config = yaml.safe_load(f)
36
+
37
+ config = normalize_config(config)
38
+
39
+ if resolve_env:
40
+ config = resolve_env_vars(
41
+ config,
42
+ strict=strict,
43
+ allow_sensitive_defaults=allow_sensitive_defaults,
44
+ )
45
+
46
+ return config
47
+
48
+
49
+ def normalize_config(config_data):
50
+ """Normalize a pgmonkey configuration dictionary.
51
+
52
+ In pgmonkey v3.0.0, the top-level 'postgresql:' wrapper key was removed.
53
+ This function detects the old format and unwraps it with a deprecation warning.
54
+
55
+ Args:
56
+ config_data: Configuration dictionary (old or new format).
57
+
58
+ Returns:
59
+ The normalized configuration dictionary (without the 'postgresql:' wrapper).
60
+ """
61
+ if isinstance(config_data, dict) and 'postgresql' in config_data:
62
+ warnings.warn(
63
+ "The top-level 'postgresql:' key in pgmonkey config files is deprecated "
64
+ "since v3.0.0 and will be removed in a future version. "
65
+ "Remove the 'postgresql:' wrapper and dedent all settings one level. "
66
+ "See https://pgmonkey.net/reference.html for the new format.",
67
+ DeprecationWarning,
68
+ stacklevel=3,
69
+ )
70
+ return config_data['postgresql']
71
+ return config_data