nui-python-shared-utils 2.4__py3-none-any.whl → 2.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/METADATA +4 -5
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/RECORD +14 -14
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/WHEEL +1 -1
- nui_shared_utils/config.py +3 -1
- nui_shared_utils/db_client.py +7 -86
- nui_shared_utils/es_client.py +15 -17
- nui_shared_utils/logging/binding.py +0 -1
- nui_shared_utils/powertools_helpers.py +2 -2
- nui_shared_utils/secrets_helper.py +174 -48
- nui_shared_utils/slack_client.py +7 -8
- nui_shared_utils/timezone.py +4 -4
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/entry_points.txt +0 -0
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/licenses/LICENSE +0 -0
- {nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nui-python-shared-utils
|
|
3
|
-
Version: 2.4
|
|
3
|
+
Version: 2.4.2
|
|
4
4
|
Summary: Shared Python utilities for AWS Lambda, CLI tools, and agents with Slack, Elasticsearch, and monitoring integrations
|
|
5
5
|
Home-page: https://github.com/nuimarkets/nui-python-shared-utils
|
|
6
6
|
Author: NUI Markets
|
|
@@ -36,7 +36,6 @@ Classifier: Intended Audience :: Developers
|
|
|
36
36
|
Classifier: License :: OSI Approved :: MIT License
|
|
37
37
|
Classifier: Operating System :: OS Independent
|
|
38
38
|
Classifier: Programming Language :: Python :: 3
|
|
39
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
40
39
|
Classifier: Programming Language :: Python :: 3.10
|
|
41
40
|
Classifier: Programming Language :: Python :: 3.11
|
|
42
41
|
Classifier: Programming Language :: Python :: 3.12
|
|
@@ -45,7 +44,7 @@ Classifier: Topic :: System :: Monitoring
|
|
|
45
44
|
Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content
|
|
46
45
|
Classifier: Environment :: Console
|
|
47
46
|
Classifier: Framework :: AWS CDK
|
|
48
|
-
Requires-Python: >=3.
|
|
47
|
+
Requires-Python: >=3.10
|
|
49
48
|
Description-Content-Type: text/markdown
|
|
50
49
|
License-File: LICENSE
|
|
51
50
|
Requires-Dist: boto3>=1.20.0
|
|
@@ -102,7 +101,7 @@ Dynamic: requires-python
|
|
|
102
101
|
# NUI Python Shared Utilities
|
|
103
102
|
|
|
104
103
|
[](https://badge.fury.io/py/nui-python-shared-utils)
|
|
105
|
-
[](https://www.python.org/downloads/)
|
|
106
105
|
[](https://opensource.org/licenses/MIT)
|
|
107
106
|
|
|
108
107
|
Production-ready shared Python utilities for AWS Lambda functions, CLI tools, and agents. Provides standardized integrations for Slack, Elasticsearch, database, and monitoring. Built and battle-tested on the NUI platform with sensible defaults configurable for any environment.
|
|
@@ -222,7 +221,7 @@ from nui_shared_utils import get_secret, get_slack_credentials
|
|
|
222
221
|
api_keys = get_secret("my-service/api-keys")
|
|
223
222
|
|
|
224
223
|
# Specialized getters with normalized field names
|
|
225
|
-
slack_creds = get_slack_credentials() #
|
|
224
|
+
slack_creds = get_slack_credentials() # SLACK_BOT_TOKEN if set, else the configured secret
|
|
226
225
|
```
|
|
227
226
|
|
|
228
227
|
**[→ See full secrets management guide](docs/getting-started/configuration.md#aws-secrets-manager)**
|
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
nui_python_shared_utils-2.4.dist-info/licenses/LICENSE,sha256=vGe2mC5yLUb8toYlY3T36ZwCB5zQUW5hlCtEMiqokhM,1071
|
|
1
|
+
nui_python_shared_utils-2.4.2.dist-info/licenses/LICENSE,sha256=vGe2mC5yLUb8toYlY3T36ZwCB5zQUW5hlCtEMiqokhM,1071
|
|
2
2
|
nui_shared_utils/__init__.py,sha256=wbm10x7THNbrCBGClWiWSVkTNxcBKcHZ-HjYpRQBFrk,13354
|
|
3
3
|
nui_shared_utils/base_client.py,sha256=I1lKQGhrKyvujV2zps0TrtEdPDqMCwRQaIh6ceGONXQ,11016
|
|
4
4
|
nui_shared_utils/cli.py,sha256=JJpSoQWKvAz4b8cO30yFNi5vY9jmqrCHzbFpvrVTkbU,8747
|
|
5
5
|
nui_shared_utils/cloudwatch_metrics.py,sha256=p0J-s63UJDGK14TRGt0k8yoj7aWSSM55UKdgcuaT1dU,12098
|
|
6
|
-
nui_shared_utils/config.py,sha256=
|
|
7
|
-
nui_shared_utils/db_client.py,sha256=
|
|
6
|
+
nui_shared_utils/config.py,sha256=heM-Zz-xKCAj6mQuhgKjfca3L6jws9Uwd2HiD1ehRXA,4704
|
|
7
|
+
nui_shared_utils/db_client.py,sha256=cm3AM0OjCejE8ryxwXdgclXlyRQ0FZUr5nWUWHfLwVo,19444
|
|
8
8
|
nui_shared_utils/error_handler.py,sha256=pJ6b_mI0Ait8QRbs9UmLzYUN-Ft0l_dnpudVO_6LRSE,12171
|
|
9
|
-
nui_shared_utils/es_client.py,sha256=
|
|
9
|
+
nui_shared_utils/es_client.py,sha256=9iyoYahpycIOHs0TEchRQKVUDCjhSwV7nU7L_SgzgXU,16969
|
|
10
10
|
nui_shared_utils/es_query_builder.py,sha256=UuheQf8b2UXaOiJlYCKYIfKfVERDZ3ag1P0OOOWm9do,12158
|
|
11
11
|
nui_shared_utils/jwt_auth.py,sha256=2Ag1zZKxd2R8QBz3aQA4h8OtKHQvUVKrjUp6lXpJxJU,9143
|
|
12
12
|
nui_shared_utils/lambda_helpers.py,sha256=psHVotpmOfnmyQCoOt4MSIEj7VwWcy6gZM3vYSZwjOk,3041
|
|
13
13
|
nui_shared_utils/llm.py,sha256=adNUO8Qbeb3__GvpROxWSUERkpiazKiGJFE5xzU5eZ0,8351
|
|
14
14
|
nui_shared_utils/log_processors.py,sha256=x5gz1LEkbmoMCZ3ZB6q-mbn26dSzwh1trz4wSkBGxqk,5538
|
|
15
|
-
nui_shared_utils/powertools_helpers.py,sha256=
|
|
16
|
-
nui_shared_utils/secrets_helper.py,sha256=
|
|
17
|
-
nui_shared_utils/slack_client.py,sha256=
|
|
15
|
+
nui_shared_utils/powertools_helpers.py,sha256=y18dT648_go5YJcI3-w-zPH2qR7cCdKlzJHm0R1HK68,13016
|
|
16
|
+
nui_shared_utils/secrets_helper.py,sha256=0dXfHAgGxYhM5tuChFi9Vz7JzJfu1slukjTIacN0IFY,11747
|
|
17
|
+
nui_shared_utils/slack_client.py,sha256=RLux3n_VrNcAtjEvR382IMOy5d34zumbJ2_VZ5zNXwk,40425
|
|
18
18
|
nui_shared_utils/slack_formatter.py,sha256=QT9VXApE0q09n_PmEVvaLCSfLW24vmowlCXF3Dzimfk,10850
|
|
19
19
|
nui_shared_utils/snowflake_client.py,sha256=e3jH6_2mMqbtP1TE8-mUFFYrfGW9eqUPT5tlcgeCqx8,16181
|
|
20
|
-
nui_shared_utils/timezone.py,sha256=
|
|
20
|
+
nui_shared_utils/timezone.py,sha256=m-Ilx9FI2Bwv-GZOZy71N_S94g26UK68jP8QEEKNw74,3867
|
|
21
21
|
nui_shared_utils/utils.py,sha256=gOpw80tZGC24QsyesR8EySRODY-TFXG8cXUvggPBsRI,8184
|
|
22
22
|
nui_shared_utils/logging/__init__.py,sha256=gVC853q3ja4RhcBVbf5jKO10lFX4upfsTvkKZx4wu54,1805
|
|
23
|
-
nui_shared_utils/logging/binding.py,sha256=
|
|
23
|
+
nui_shared_utils/logging/binding.py,sha256=FJhC2RjzhTczWz43Bk1HiQGsorSVFVHvmNmPhbwlW78,13495
|
|
24
24
|
nui_shared_utils/logging/fields.py,sha256=7jhBdyEXAJod61aKPCk68idECTAV-_BNjiABp9IRFZo,11194
|
|
25
25
|
nui_shared_utils/logging/redaction.py,sha256=t-EtNhR1nK6IOae70GnieHxn_36_jYuuta32T1dMlP8,6928
|
|
26
26
|
nui_shared_utils/slack_setup/__init__.py,sha256=OElyS3xk4F_YKH5uUUTDpN0ah1dOO3e52muIPjAMPW8,320
|
|
27
27
|
nui_shared_utils/slack_setup/channel_creator.py,sha256=0gyCBIS0EC96SVn1Z2dD4Fpzk0xNdHSWyiCxoUQIUe8,10667
|
|
28
28
|
nui_shared_utils/slack_setup/channel_definitions.py,sha256=atfz5ZhpqefOeLh1gShbWd-TLzgjPmhv95bfuTdmZog,5676
|
|
29
29
|
nui_shared_utils/slack_setup/setup_helpers.py,sha256=pzzXMs12GI9sdZttWeYzgLPgC0xqPz7ZLHM1GUPNNXc,7219
|
|
30
|
-
nui_python_shared_utils-2.4.dist-info/METADATA,sha256=
|
|
31
|
-
nui_python_shared_utils-2.4.dist-info/WHEEL,sha256=
|
|
32
|
-
nui_python_shared_utils-2.4.dist-info/entry_points.txt,sha256=TrJ3Z4kz3oXfy6InXn20jHu7rnTwo-4EA0KRFX3fkL0,66
|
|
33
|
-
nui_python_shared_utils-2.4.dist-info/top_level.txt,sha256=YMRu5cnn3hxHoYq9wNvaI9atVHnTNVdswO5_cQSfnJI,17
|
|
34
|
-
nui_python_shared_utils-2.4.dist-info/RECORD,,
|
|
30
|
+
nui_python_shared_utils-2.4.2.dist-info/METADATA,sha256=mmzQthtSgUtuItVwaxcgItG0L5FCRogZPLgBCpBLhF4,22666
|
|
31
|
+
nui_python_shared_utils-2.4.2.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
32
|
+
nui_python_shared_utils-2.4.2.dist-info/entry_points.txt,sha256=TrJ3Z4kz3oXfy6InXn20jHu7rnTwo-4EA0KRFX3fkL0,66
|
|
33
|
+
nui_python_shared_utils-2.4.2.dist-info/top_level.txt,sha256=YMRu5cnn3hxHoYq9wNvaI9atVHnTNVdswO5_cQSfnJI,17
|
|
34
|
+
nui_python_shared_utils-2.4.2.dist-info/RECORD,,
|
nui_shared_utils/config.py
CHANGED
|
@@ -34,7 +34,9 @@ class Config:
|
|
|
34
34
|
slack_credentials_secret: AWS secret name for Slack credentials
|
|
35
35
|
aws_region: AWS region for secrets/services
|
|
36
36
|
"""
|
|
37
|
-
# Elasticsearch settings
|
|
37
|
+
# Elasticsearch settings. An es_host passed here outranks a host carried in a
|
|
38
|
+
# credentials secret; the env-derived and built-in values do not.
|
|
39
|
+
self._es_host_explicit = es_host is not None
|
|
38
40
|
if es_host is not None:
|
|
39
41
|
self.es_host = es_host
|
|
40
42
|
else:
|
nui_shared_utils/db_client.py
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
Refactored Database client using BaseClient for DRY code patterns.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
import os
|
|
6
5
|
import time
|
|
7
6
|
import logging
|
|
8
7
|
from typing import Dict, List, Optional, Any
|
|
@@ -11,7 +10,7 @@ import pymysql
|
|
|
11
10
|
|
|
12
11
|
from .base_client import BaseClient, ServiceHealthMixin
|
|
13
12
|
from .utils import handle_client_errors, safe_close_connection
|
|
14
|
-
from .secrets_helper import get_database_credentials
|
|
13
|
+
from .secrets_helper import get_database_credentials, _db_credentials_from_env, _normalise_db_secret
|
|
15
14
|
|
|
16
15
|
# Optional PostgreSQL support
|
|
17
16
|
try:
|
|
@@ -27,47 +26,6 @@ log = logging.getLogger(__name__)
|
|
|
27
26
|
_connection_pool = {}
|
|
28
27
|
|
|
29
28
|
|
|
30
|
-
def _clean_expired_connections(pool_key: str, pool_recycle: int) -> None:
|
|
31
|
-
"""
|
|
32
|
-
Clean expired connections from the pool.
|
|
33
|
-
|
|
34
|
-
Args:
|
|
35
|
-
pool_key: Pool identifier
|
|
36
|
-
pool_recycle: Maximum connection age in seconds (0 disables recycling)
|
|
37
|
-
"""
|
|
38
|
-
if not pool_recycle or pool_recycle <= 0:
|
|
39
|
-
return
|
|
40
|
-
|
|
41
|
-
if pool_key not in _connection_pool:
|
|
42
|
-
return
|
|
43
|
-
|
|
44
|
-
current_time = time.time()
|
|
45
|
-
pool_entries = _connection_pool[pool_key]
|
|
46
|
-
|
|
47
|
-
# Find expired connections
|
|
48
|
-
expired_connections = []
|
|
49
|
-
fresh_connections = []
|
|
50
|
-
|
|
51
|
-
for entry in pool_entries:
|
|
52
|
-
age = current_time - entry["timestamp"]
|
|
53
|
-
if age > pool_recycle:
|
|
54
|
-
expired_connections.append(entry["connection"])
|
|
55
|
-
else:
|
|
56
|
-
fresh_connections.append(entry)
|
|
57
|
-
|
|
58
|
-
# Close expired connections
|
|
59
|
-
for conn in expired_connections:
|
|
60
|
-
safe_close_connection(conn)
|
|
61
|
-
|
|
62
|
-
# Update pool with only fresh connections
|
|
63
|
-
_connection_pool[pool_key] = fresh_connections
|
|
64
|
-
|
|
65
|
-
if expired_connections:
|
|
66
|
-
log.debug(
|
|
67
|
-
f"Cleaned {len(expired_connections)} expired connections from pool {pool_key}"
|
|
68
|
-
)
|
|
69
|
-
|
|
70
|
-
|
|
71
29
|
def get_pool_stats() -> Dict[str, Any]:
|
|
72
30
|
"""
|
|
73
31
|
Get current connection pool statistics.
|
|
@@ -158,26 +116,8 @@ class DatabaseClient(BaseClient, ServiceHealthMixin):
|
|
|
158
116
|
return None
|
|
159
117
|
|
|
160
118
|
def _resolve_credentials_from_env(self) -> Optional[Dict[str, Any]]:
|
|
161
|
-
"""Resolve MySQL credentials from
|
|
162
|
-
|
|
163
|
-
Requires both DB_HOST and DB_PASSWORD to trigger.
|
|
164
|
-
"""
|
|
165
|
-
host = os.environ.get("DB_HOST")
|
|
166
|
-
password = os.environ.get("DB_PASSWORD")
|
|
167
|
-
if not host or not password:
|
|
168
|
-
return None
|
|
169
|
-
port_str = os.environ.get("DB_PORT", "3306")
|
|
170
|
-
try:
|
|
171
|
-
port = int(port_str)
|
|
172
|
-
except ValueError:
|
|
173
|
-
raise ValueError(f"DB_PORT must be an integer, got: {port_str!r}")
|
|
174
|
-
return {
|
|
175
|
-
"host": host,
|
|
176
|
-
"port": port,
|
|
177
|
-
"username": os.environ.get("DB_USERNAME", "root"),
|
|
178
|
-
"password": password,
|
|
179
|
-
"database": os.environ.get("DB_DATABASE", "app"),
|
|
180
|
-
}
|
|
119
|
+
"""Resolve MySQL credentials from DB_* env vars (DB_HOST + DB_PASSWORD required)."""
|
|
120
|
+
return _db_credentials_from_env("mysql")
|
|
181
121
|
|
|
182
122
|
def _fetch_credentials_from_sm(self, secret_name: Optional[str]) -> Dict[str, Any]:
|
|
183
123
|
"""Override to use get_database_credentials for normalized field names."""
|
|
@@ -511,29 +451,11 @@ class PostgreSQLClient(BaseClient, ServiceHealthMixin):
|
|
|
511
451
|
return None
|
|
512
452
|
|
|
513
453
|
def _resolve_credentials_from_env(self) -> Optional[Dict[str, Any]]:
|
|
514
|
-
"""Resolve PostgreSQL credentials from
|
|
515
|
-
|
|
516
|
-
Requires both DB_HOST and DB_PASSWORD to trigger.
|
|
517
|
-
"""
|
|
518
|
-
host = os.environ.get("DB_HOST")
|
|
519
|
-
password = os.environ.get("DB_PASSWORD")
|
|
520
|
-
if not host or not password:
|
|
521
|
-
return None
|
|
522
|
-
port_str = os.environ.get("DB_PORT", "5432")
|
|
523
|
-
try:
|
|
524
|
-
port = int(port_str)
|
|
525
|
-
except ValueError:
|
|
526
|
-
raise ValueError(f"DB_PORT must be an integer, got: {port_str!r}")
|
|
527
|
-
return {
|
|
528
|
-
"host": host,
|
|
529
|
-
"port": port,
|
|
530
|
-
"username": os.environ.get("DB_USERNAME", "postgres"),
|
|
531
|
-
"password": password,
|
|
532
|
-
"database": os.environ.get("DB_DATABASE", "postgres"),
|
|
533
|
-
}
|
|
454
|
+
"""Resolve PostgreSQL credentials from DB_* env vars (DB_HOST + DB_PASSWORD required)."""
|
|
455
|
+
return _db_credentials_from_env("postgresql")
|
|
534
456
|
|
|
535
457
|
def _fetch_credentials_from_sm(self, secret_name: Optional[str]) -> Dict[str, Any]:
|
|
536
|
-
"""
|
|
458
|
+
"""Fetch from Secrets Manager, preferring the auth_* keys when requested and present."""
|
|
537
459
|
from .secrets_helper import get_secret
|
|
538
460
|
|
|
539
461
|
resolved_secret_name = self._resolve_secret_name(secret_name)
|
|
@@ -548,8 +470,7 @@ class PostgreSQLClient(BaseClient, ServiceHealthMixin):
|
|
|
548
470
|
"password": raw_creds.get("auth_password"),
|
|
549
471
|
"database": raw_creds.get("auth_database", "auth-service-db"),
|
|
550
472
|
}
|
|
551
|
-
|
|
552
|
-
return get_database_credentials(resolved_secret_name)
|
|
473
|
+
return _normalise_db_secret(raw_creds, "postgresql")
|
|
553
474
|
|
|
554
475
|
@contextmanager
|
|
555
476
|
def get_connection(self, database: Optional[str] = None):
|
nui_shared_utils/es_client.py
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
Refactored Elasticsearch client using BaseClient for DRY code patterns.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
-
import os
|
|
6
5
|
import logging
|
|
7
6
|
from datetime import datetime, timedelta
|
|
8
7
|
from typing import Dict, Iterator, List, Optional, Any, Tuple
|
|
@@ -11,7 +10,8 @@ from elasticsearch import Elasticsearch
|
|
|
11
10
|
from elasticsearch.helpers import streaming_bulk as es_streaming_bulk
|
|
12
11
|
|
|
13
12
|
from .base_client import BaseClient, ServiceHealthMixin
|
|
14
|
-
from .utils import handle_client_errors
|
|
13
|
+
from .utils import handle_client_errors
|
|
14
|
+
from .secrets_helper import _es_credentials_from_env, _es_host_with_port, _normalise_es_secret, _resolve_es_host
|
|
15
15
|
|
|
16
16
|
log = logging.getLogger(__name__)
|
|
17
17
|
|
|
@@ -55,29 +55,27 @@ class ElasticsearchClient(BaseClient, ServiceHealthMixin):
|
|
|
55
55
|
Accepts both conventions: ES_PASSWORD/ES_USERNAME and ES_PASS/ES_USER.
|
|
56
56
|
Password is required to trigger. Username defaults to "elastic".
|
|
57
57
|
"""
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
}
|
|
58
|
+
return _es_credentials_from_env()
|
|
59
|
+
|
|
60
|
+
def _fetch_credentials_from_sm(self, secret_name: Optional[str]) -> Dict[str, Any]:
|
|
61
|
+
"""Fetch from Secrets Manager; normalise host, username, password and keep any other keys."""
|
|
62
|
+
raw = super()._fetch_credentials_from_sm(secret_name)
|
|
63
|
+
return {**raw, **_normalise_es_secret(raw)}
|
|
65
64
|
|
|
66
65
|
def _create_service_client(self) -> Elasticsearch:
|
|
67
66
|
"""Create Elasticsearch client with resolved configuration."""
|
|
68
|
-
#
|
|
69
|
-
host
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
67
|
+
# Host precedence: constructor arg > ES_HOST/ELASTICSEARCH_HOST > configure(es_host=...)
|
|
68
|
+
# > credentials "host" (e.g. from the secret) > default. Shared with get_elasticsearch_credentials.
|
|
69
|
+
if self._host_override is not None:
|
|
70
|
+
host = self._host_override
|
|
71
|
+
else:
|
|
72
|
+
host = _resolve_es_host(self.credentials.get("host"))
|
|
73
|
+
host = _es_host_with_port(host)
|
|
74
74
|
|
|
75
75
|
parsed = urlparse(host)
|
|
76
76
|
if parsed.scheme and parsed.netloc: # Valid URL with scheme://netloc
|
|
77
77
|
es_url = host
|
|
78
78
|
else:
|
|
79
|
-
if ":" not in host:
|
|
80
|
-
host = f"{host}:9200"
|
|
81
79
|
scheme = self.client_config.get("scheme", "http")
|
|
82
80
|
es_url = f"{scheme}://{host}"
|
|
83
81
|
|
|
@@ -306,7 +306,6 @@ def log_exception(binder: Any, exc: BaseException, message: str) -> None:
|
|
|
306
306
|
extra={
|
|
307
307
|
F.ERROR_TYPE: type(exc).__name__,
|
|
308
308
|
F.ERROR_MESSAGE: str(exc),
|
|
309
|
-
# 3-arg form for Python 3.9 compatibility (single-arg added in 3.10).
|
|
310
309
|
F.ERROR_STACK: "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
|
|
311
310
|
},
|
|
312
311
|
)
|
|
@@ -75,7 +75,7 @@ def get_powertools_logger(
|
|
|
75
75
|
and enforces UTC timezone for consistency with log aggregation systems.
|
|
76
76
|
|
|
77
77
|
Args:
|
|
78
|
-
service_name: Service identifier (e.g., "
|
|
78
|
+
service_name: Service identifier (e.g., "order-processor", "email-ingest")
|
|
79
79
|
level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Default: INFO
|
|
80
80
|
local_dev_colors: Enable coloredlogs for local development. Default: True
|
|
81
81
|
|
|
@@ -182,7 +182,7 @@ def powertools_handler(
|
|
|
182
182
|
|
|
183
183
|
Args:
|
|
184
184
|
service_name: Service identifier for logging and metrics dimensions
|
|
185
|
-
metrics_namespace: CloudWatch namespace for metrics (e.g., "
|
|
185
|
+
metrics_namespace: CloudWatch namespace for metrics (e.g., "Ecommerce/Orders").
|
|
186
186
|
If None, metrics publishing is disabled.
|
|
187
187
|
slack_alert_channel: Slack channel for error alerts (e.g., "#alerts", "#errors").
|
|
188
188
|
If None, Slack alerting is disabled.
|
|
@@ -6,7 +6,8 @@ Shared across all AWS Lambda functions.
|
|
|
6
6
|
import os
|
|
7
7
|
import json
|
|
8
8
|
import logging
|
|
9
|
-
from
|
|
9
|
+
from urllib.parse import urlparse
|
|
10
|
+
from typing import Any, Dict, Optional
|
|
10
11
|
|
|
11
12
|
from .config import get_config
|
|
12
13
|
|
|
@@ -86,41 +87,180 @@ def get_secret(secret_name: str) -> Dict:
|
|
|
86
87
|
raise Exception(f"Unexpected error retrieving secret {secret_name}: {str(e)}")
|
|
87
88
|
|
|
88
89
|
|
|
90
|
+
# Per-engine defaults for database credentials. The env tier fills all three;
|
|
91
|
+
# the Secrets Manager tier fills port and database only, so a secret missing its
|
|
92
|
+
# username stays visibly broken rather than borrowing a guessed one.
|
|
93
|
+
_DB_DEFAULTS: Dict[str, Dict[str, Any]] = {
|
|
94
|
+
"mysql": {"port": 3306, "username": "root", "database": "app"},
|
|
95
|
+
"postgresql": {"port": 5432, "username": "postgres", "database": "postgres"},
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _db_credentials_from_env(engine: str = "mysql") -> Optional[Dict[str, Any]]:
|
|
100
|
+
"""
|
|
101
|
+
Resolve database credentials from DB_* environment variables.
|
|
102
|
+
|
|
103
|
+
Both DB_HOST and DB_PASSWORD must be set; otherwise returns None so the
|
|
104
|
+
caller falls through to Secrets Manager. Shared by the database clients and
|
|
105
|
+
get_database_credentials.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
engine: Key into the per-engine defaults ("mysql" or "postgresql")
|
|
109
|
+
|
|
110
|
+
Raises:
|
|
111
|
+
ValueError: DB_PORT is set but is not an integer
|
|
112
|
+
"""
|
|
113
|
+
host = os.environ.get("DB_HOST")
|
|
114
|
+
password = os.environ.get("DB_PASSWORD")
|
|
115
|
+
if not host or not password:
|
|
116
|
+
return None
|
|
117
|
+
defaults = _DB_DEFAULTS[engine]
|
|
118
|
+
port_str = os.environ.get("DB_PORT", str(defaults["port"]))
|
|
119
|
+
try:
|
|
120
|
+
port = int(port_str)
|
|
121
|
+
except ValueError:
|
|
122
|
+
raise ValueError(f"DB_PORT must be an integer, got: {port_str!r}")
|
|
123
|
+
return {
|
|
124
|
+
"host": host,
|
|
125
|
+
"port": port,
|
|
126
|
+
"username": os.environ.get("DB_USERNAME", defaults["username"]),
|
|
127
|
+
"password": password,
|
|
128
|
+
"database": os.environ.get("DB_DATABASE", defaults["database"]),
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _normalise_db_secret(creds: Dict[str, Any], engine: str = "mysql") -> Dict[str, Any]:
|
|
133
|
+
"""
|
|
134
|
+
Map a database secret onto host, port, username, password, database.
|
|
135
|
+
|
|
136
|
+
Accepts the common alternative field names (endpoint/hostname, user, dbname)
|
|
137
|
+
and fills a missing port or database from the engine's defaults.
|
|
138
|
+
"""
|
|
139
|
+
defaults = _DB_DEFAULTS[engine]
|
|
140
|
+
return {
|
|
141
|
+
"host": creds.get("host", creds.get("endpoint", creds.get("hostname"))),
|
|
142
|
+
"port": int(creds.get("port", defaults["port"])),
|
|
143
|
+
"username": creds.get("username", creds.get("user")),
|
|
144
|
+
"password": creds.get("password"),
|
|
145
|
+
"database": creds.get("database", creds.get("dbname", defaults["database"])),
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _es_credentials_from_env() -> Optional[Dict[str, Any]]:
|
|
150
|
+
"""
|
|
151
|
+
Resolve Elasticsearch username and password from environment variables.
|
|
152
|
+
|
|
153
|
+
Accepts ES_PASSWORD/ES_USERNAME and ES_PASS/ES_USER. A password is required
|
|
154
|
+
to trigger; the username defaults to "elastic". Returns None otherwise.
|
|
155
|
+
"""
|
|
156
|
+
password = os.environ.get("ES_PASSWORD") or os.environ.get("ES_PASS")
|
|
157
|
+
if not password:
|
|
158
|
+
return None
|
|
159
|
+
return {
|
|
160
|
+
"username": os.environ.get("ES_USERNAME") or os.environ.get("ES_USER") or "elastic",
|
|
161
|
+
"password": password,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _normalise_es_secret(creds: Dict[str, Any]) -> Dict[str, Any]:
|
|
166
|
+
"""Map an Elasticsearch secret onto host, username, password (host may be None)."""
|
|
167
|
+
return {
|
|
168
|
+
"host": creds.get("host"),
|
|
169
|
+
"username": creds.get("username", "elastic"),
|
|
170
|
+
"password": creds.get("password"),
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _resolve_es_host(credentials_host: Optional[str] = None) -> str:
|
|
175
|
+
"""
|
|
176
|
+
Pick the Elasticsearch host: ES_HOST (or ELASTICSEARCH_HOST), then a host set
|
|
177
|
+
explicitly through configure(es_host=...) or Config(es_host=...), then the host
|
|
178
|
+
carried by the resolved credentials, then the configured default (localhost:9200).
|
|
179
|
+
"""
|
|
180
|
+
env_host = os.environ.get("ES_HOST") or os.environ.get("ELASTICSEARCH_HOST")
|
|
181
|
+
if env_host:
|
|
182
|
+
return env_host
|
|
183
|
+
config = get_config()
|
|
184
|
+
if getattr(config, "_es_host_explicit", False) is True:
|
|
185
|
+
return config.es_host
|
|
186
|
+
return credentials_host or config.es_host
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _es_host_with_port(host: str) -> str:
|
|
190
|
+
"""
|
|
191
|
+
Return ``host`` unchanged when it is a URL (scheme://netloc) or already names a
|
|
192
|
+
port; otherwise append the default port ``:9200``.
|
|
193
|
+
"""
|
|
194
|
+
parsed = urlparse(host)
|
|
195
|
+
if parsed.scheme and parsed.netloc:
|
|
196
|
+
return host
|
|
197
|
+
if ":" not in host:
|
|
198
|
+
return f"{host}:9200"
|
|
199
|
+
return host
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _slack_credentials_from_env() -> Optional[Dict[str, Any]]:
|
|
203
|
+
"""
|
|
204
|
+
Resolve Slack credentials from SLACK_BOT_TOKEN (required to trigger) and
|
|
205
|
+
SLACK_WEBHOOK_URL (optional). Returns None when no bot token is set.
|
|
206
|
+
"""
|
|
207
|
+
bot_token = os.environ.get("SLACK_BOT_TOKEN")
|
|
208
|
+
if not bot_token:
|
|
209
|
+
return None
|
|
210
|
+
return {"bot_token": bot_token, "webhook_url": os.environ.get("SLACK_WEBHOOK_URL") or None}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _normalise_slack_secret(creds: Dict[str, Any]) -> Dict[str, Any]:
|
|
214
|
+
"""Map a Slack secret onto bot_token (or legacy token) and optional webhook_url.
|
|
215
|
+
|
|
216
|
+
An empty or null ``bot_token`` falls back to ``token``.
|
|
217
|
+
"""
|
|
218
|
+
return {
|
|
219
|
+
"bot_token": creds.get("bot_token") or creds.get("token"),
|
|
220
|
+
"webhook_url": creds.get("webhook_url"),
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
|
|
89
224
|
def get_database_credentials(secret_name: Optional[str] = None) -> Dict:
|
|
90
225
|
"""
|
|
91
|
-
Get database credentials with standardized field names.
|
|
226
|
+
Get MySQL database credentials with standardized field names.
|
|
227
|
+
|
|
228
|
+
Precedence (matching DatabaseClient):
|
|
229
|
+
1. Environment variables (DB_HOST + DB_PASSWORD, optional DB_PORT,
|
|
230
|
+
DB_USERNAME, DB_DATABASE)
|
|
231
|
+
2. AWS Secrets Manager
|
|
92
232
|
|
|
93
233
|
Args:
|
|
94
|
-
secret_name: Override default
|
|
234
|
+
secret_name: Override default secret name for Secrets Manager lookup.
|
|
235
|
+
Ignored when credentials are resolved from environment variables.
|
|
95
236
|
|
|
96
237
|
Returns:
|
|
97
238
|
Dict with host, port, username, password, database
|
|
98
239
|
"""
|
|
240
|
+
env_creds = _db_credentials_from_env("mysql")
|
|
241
|
+
if env_creds is not None:
|
|
242
|
+
return env_creds
|
|
243
|
+
|
|
99
244
|
config = get_config()
|
|
100
245
|
secret = secret_name or os.environ.get("DB_CREDENTIALS_SECRET") or config.db_credentials_secret
|
|
101
246
|
if not secret:
|
|
102
247
|
raise ValueError("No database secret name provided")
|
|
103
248
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
# Normalize field names
|
|
107
|
-
return {
|
|
108
|
-
"host": creds.get("host", creds.get("endpoint", creds.get("hostname"))),
|
|
109
|
-
"port": int(creds.get("port", 3306)),
|
|
110
|
-
"username": creds.get("username", creds.get("user")),
|
|
111
|
-
"password": creds.get("password"),
|
|
112
|
-
"database": creds.get("database", creds.get("dbname", "app")),
|
|
113
|
-
}
|
|
249
|
+
return _normalise_db_secret(get_secret(secret), "mysql")
|
|
114
250
|
|
|
115
251
|
|
|
116
252
|
def get_elasticsearch_credentials(secret_name: Optional[str] = None) -> Dict:
|
|
117
253
|
"""
|
|
118
254
|
Get Elasticsearch credentials.
|
|
119
255
|
|
|
120
|
-
Precedence (matching
|
|
256
|
+
Precedence (matching ElasticsearchClient):
|
|
121
257
|
1. Environment variables (ES_PASSWORD/ES_PASS + ES_USERNAME/ES_USER)
|
|
122
258
|
2. AWS Secrets Manager
|
|
123
259
|
|
|
260
|
+
The host is ES_HOST (or ELASTICSEARCH_HOST), then a host set with
|
|
261
|
+
configure(es_host=...), then the secret's ``host`` field, then the default,
|
|
262
|
+
with ``:9200`` appended when no port is given and it is not a URL.
|
|
263
|
+
|
|
124
264
|
Args:
|
|
125
265
|
secret_name: Override default secret name for Secrets Manager lookup.
|
|
126
266
|
Ignored when credentials are resolved from environment variables.
|
|
@@ -128,58 +268,44 @@ def get_elasticsearch_credentials(secret_name: Optional[str] = None) -> Dict:
|
|
|
128
268
|
Returns:
|
|
129
269
|
Dict with host, username, password
|
|
130
270
|
"""
|
|
131
|
-
|
|
271
|
+
creds = _es_credentials_from_env()
|
|
272
|
+
if creds is None:
|
|
273
|
+
config = get_config()
|
|
274
|
+
secret = secret_name or os.environ.get("ES_CREDENTIALS_SECRET") or config.es_credentials_secret
|
|
275
|
+
if not secret:
|
|
276
|
+
raise ValueError("No Elasticsearch secret name provided")
|
|
277
|
+
creds = _normalise_es_secret(get_secret(secret))
|
|
132
278
|
|
|
133
|
-
|
|
134
|
-
es_password = os.environ.get("ES_PASSWORD") or os.environ.get("ES_PASS")
|
|
135
|
-
if es_password:
|
|
136
|
-
es_user = os.environ.get("ES_USERNAME") or os.environ.get("ES_USER") or "elastic"
|
|
137
|
-
host = os.environ.get("ES_HOST") or config.es_host
|
|
138
|
-
if ":" not in host and not host.startswith("http"):
|
|
139
|
-
host = f"{host}:9200"
|
|
140
|
-
return {"host": host, "username": es_user, "password": es_password}
|
|
141
|
-
|
|
142
|
-
# Secrets Manager (Lambda runtime)
|
|
143
|
-
secret = secret_name or os.environ.get("ES_CREDENTIALS_SECRET") or config.es_credentials_secret
|
|
144
|
-
if not secret:
|
|
145
|
-
raise ValueError("No Elasticsearch secret name provided")
|
|
279
|
+
host = _es_host_with_port(_resolve_es_host(creds.get("host")))
|
|
146
280
|
|
|
147
|
-
creds
|
|
148
|
-
|
|
149
|
-
# Use configuration system for host defaults instead of hardcoded value
|
|
150
|
-
host = os.environ.get("ES_HOST") or creds.get("host") or config.es_host
|
|
151
|
-
# Ensure port is included if not already present
|
|
152
|
-
if ":" not in host and not host.startswith("http"):
|
|
153
|
-
host = f"{host}:9200"
|
|
154
|
-
|
|
155
|
-
return {
|
|
156
|
-
"host": host,
|
|
157
|
-
"username": creds.get("username", "elastic"),
|
|
158
|
-
"password": creds.get("password"),
|
|
159
|
-
}
|
|
281
|
+
return {"host": host, "username": creds["username"], "password": creds["password"]}
|
|
160
282
|
|
|
161
283
|
|
|
162
284
|
def get_slack_credentials(secret_name: Optional[str] = None) -> Dict:
|
|
163
285
|
"""
|
|
164
286
|
Get Slack bot credentials.
|
|
165
287
|
|
|
288
|
+
Precedence (matching SlackClient):
|
|
289
|
+
1. Environment variables (SLACK_BOT_TOKEN, optional SLACK_WEBHOOK_URL)
|
|
290
|
+
2. AWS Secrets Manager
|
|
291
|
+
|
|
166
292
|
Args:
|
|
167
|
-
secret_name: Override default
|
|
293
|
+
secret_name: Override default secret name for Secrets Manager lookup.
|
|
294
|
+
Ignored when credentials are resolved from environment variables.
|
|
168
295
|
|
|
169
296
|
Returns:
|
|
170
297
|
Dict with bot_token and optional webhook_url
|
|
171
298
|
"""
|
|
299
|
+
env_creds = _slack_credentials_from_env()
|
|
300
|
+
if env_creds is not None:
|
|
301
|
+
return env_creds
|
|
302
|
+
|
|
172
303
|
config = get_config()
|
|
173
304
|
secret = secret_name or os.environ.get("SLACK_CREDENTIALS_SECRET") or config.slack_credentials_secret
|
|
174
305
|
if not secret:
|
|
175
306
|
raise ValueError("No Slack secret name provided")
|
|
176
307
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
return {
|
|
180
|
-
"bot_token": creds.get("bot_token", creds.get("token")),
|
|
181
|
-
"webhook_url": creds.get("webhook_url"), # Optional
|
|
182
|
-
}
|
|
308
|
+
return _normalise_slack_secret(get_secret(secret))
|
|
183
309
|
|
|
184
310
|
|
|
185
311
|
def get_api_key(secret_name: str, key_field: str = "api_key") -> str:
|
nui_shared_utils/slack_client.py
CHANGED
|
@@ -19,6 +19,7 @@ except ImportError:
|
|
|
19
19
|
YAML_AVAILABLE = False
|
|
20
20
|
|
|
21
21
|
from .base_client import BaseClient, ServiceHealthMixin
|
|
22
|
+
from .secrets_helper import _normalise_slack_secret, _slack_credentials_from_env
|
|
22
23
|
from .utils import create_aws_client, handle_client_errors
|
|
23
24
|
from .lambda_helpers import get_lambda_environment_info
|
|
24
25
|
|
|
@@ -258,14 +259,12 @@ class SlackClient(BaseClient, ServiceHealthMixin):
|
|
|
258
259
|
Checks for SLACK_BOT_TOKEN (required to trigger).
|
|
259
260
|
Optionally picks up SLACK_WEBHOOK_URL.
|
|
260
261
|
"""
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
creds["webhook_url"] = webhook_url
|
|
268
|
-
return creds
|
|
262
|
+
return _slack_credentials_from_env()
|
|
263
|
+
|
|
264
|
+
def _fetch_credentials_from_sm(self, secret_name: Optional[str]) -> Dict[str, Any]:
|
|
265
|
+
"""Fetch from Secrets Manager; normalise bot_token, webhook_url and keep any other keys."""
|
|
266
|
+
raw = super()._fetch_credentials_from_sm(secret_name)
|
|
267
|
+
return {**raw, **_normalise_slack_secret(raw)}
|
|
269
268
|
|
|
270
269
|
def _create_service_client(self) -> WebClient:
|
|
271
270
|
"""Create Slack WebClient with credentials."""
|
nui_shared_utils/timezone.py
CHANGED
|
@@ -13,8 +13,8 @@ from zoneinfo import ZoneInfo
|
|
|
13
13
|
# Supported timezones
|
|
14
14
|
NZ_TZ = ZoneInfo("Pacific/Auckland") # Primary timezone
|
|
15
15
|
AUS_TZ = ZoneInfo("Australia/Sydney")
|
|
16
|
-
US_EAST_TZ = ZoneInfo("
|
|
17
|
-
US_WEST_TZ = ZoneInfo("
|
|
16
|
+
US_EAST_TZ = ZoneInfo("America/New_York")
|
|
17
|
+
US_WEST_TZ = ZoneInfo("America/Los_Angeles")
|
|
18
18
|
EUR_TZ = ZoneInfo("Europe/London")
|
|
19
19
|
|
|
20
20
|
|
|
@@ -115,7 +115,7 @@ def get_timezone_options() -> List[Tuple[str, str]]:
|
|
|
115
115
|
return [
|
|
116
116
|
("nz", "New Zealand (Pacific/Auckland)"),
|
|
117
117
|
("aus", "Australia (Australia/Sydney)"),
|
|
118
|
-
("us_east", "US Eastern (
|
|
119
|
-
("us_west", "US Pacific (
|
|
118
|
+
("us_east", "US Eastern (America/New_York)"),
|
|
119
|
+
("us_west", "US Pacific (America/Los_Angeles)"),
|
|
120
120
|
("eur", "Europe (Europe/London)"),
|
|
121
121
|
]
|
{nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{nui_python_shared_utils-2.4.dist-info → nui_python_shared_utils-2.4.2.dist-info}/top_level.txt
RENAMED
|
File without changes
|