runbooks 1.1.9__py3-none-any.whl → 1.1.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/__init___optimized.py +2 -1
- runbooks/_platform/__init__.py +1 -1
- runbooks/cfat/cli.py +4 -3
- runbooks/cfat/cloud_foundations_assessment.py +1 -2
- runbooks/cfat/tests/test_cli.py +4 -1
- runbooks/cli/commands/finops.py +68 -19
- runbooks/cli/commands/inventory.py +796 -7
- runbooks/cli/commands/operate.py +65 -4
- runbooks/cloudops/cost_optimizer.py +1 -3
- runbooks/common/cli_decorators.py +6 -4
- runbooks/common/config_loader.py +787 -0
- runbooks/common/config_schema.py +280 -0
- runbooks/common/dry_run_framework.py +14 -2
- runbooks/common/mcp_integration.py +238 -0
- runbooks/finops/ebs_cost_optimizer.py +7 -4
- runbooks/finops/elastic_ip_optimizer.py +7 -4
- runbooks/finops/infrastructure/__init__.py +3 -2
- runbooks/finops/infrastructure/commands.py +7 -4
- runbooks/finops/infrastructure/load_balancer_optimizer.py +7 -4
- runbooks/finops/infrastructure/vpc_endpoint_optimizer.py +7 -4
- runbooks/finops/nat_gateway_optimizer.py +7 -4
- runbooks/finops/tests/run_tests.py +1 -1
- runbooks/inventory/ArgumentsClass.py +2 -1
- runbooks/inventory/README.md +111 -12
- runbooks/inventory/Tests/test_Inventory_Modules.py +27 -10
- runbooks/inventory/Tests/test_cfn_describe_stacks.py +18 -7
- runbooks/inventory/Tests/test_ec2_describe_instances.py +30 -15
- runbooks/inventory/Tests/test_lambda_list_functions.py +17 -3
- runbooks/inventory/Tests/test_org_list_accounts.py +17 -4
- runbooks/inventory/account_class.py +0 -1
- runbooks/inventory/all_my_instances_wrapper.py +4 -8
- runbooks/inventory/aws_organization.png +0 -0
- runbooks/inventory/check_cloudtrail_compliance.py +4 -4
- runbooks/inventory/check_controltower_readiness.py +50 -47
- runbooks/inventory/check_landingzone_readiness.py +35 -31
- runbooks/inventory/cloud_foundations_integration.py +8 -3
- runbooks/inventory/core/collector.py +201 -1
- runbooks/inventory/discovery.md +2 -1
- runbooks/inventory/{draw_org_structure.py → draw_org.py} +55 -9
- runbooks/inventory/drift_detection_cli.py +8 -68
- runbooks/inventory/find_cfn_drift_detection.py +14 -4
- runbooks/inventory/find_cfn_orphaned_stacks.py +7 -5
- runbooks/inventory/find_cfn_stackset_drift.py +5 -5
- runbooks/inventory/find_ec2_security_groups.py +6 -3
- runbooks/inventory/find_landingzone_versions.py +5 -5
- runbooks/inventory/find_vpc_flow_logs.py +5 -5
- runbooks/inventory/inventory.sh +20 -7
- runbooks/inventory/inventory_mcp_cli.py +4 -0
- runbooks/inventory/inventory_modules.py +9 -7
- runbooks/inventory/list_cfn_stacks.py +18 -8
- runbooks/inventory/list_cfn_stackset_operation_results.py +2 -2
- runbooks/inventory/list_cfn_stackset_operations.py +32 -20
- runbooks/inventory/list_cfn_stacksets.py +7 -4
- runbooks/inventory/list_config_recorders_delivery_channels.py +4 -4
- runbooks/inventory/list_ds_directories.py +3 -3
- runbooks/inventory/list_ec2_availability_zones.py +7 -3
- runbooks/inventory/list_ec2_ebs_volumes.py +3 -3
- runbooks/inventory/list_ec2_instances.py +1 -1
- runbooks/inventory/list_ecs_clusters_and_tasks.py +8 -4
- runbooks/inventory/list_elbs_load_balancers.py +7 -3
- runbooks/inventory/list_enis_network_interfaces.py +3 -3
- runbooks/inventory/list_guardduty_detectors.py +9 -5
- runbooks/inventory/list_iam_policies.py +7 -3
- runbooks/inventory/list_iam_roles.py +3 -3
- runbooks/inventory/list_iam_saml_providers.py +8 -4
- runbooks/inventory/list_lambda_functions.py +8 -4
- runbooks/inventory/list_org_accounts.py +306 -276
- runbooks/inventory/list_org_accounts_users.py +45 -9
- runbooks/inventory/list_rds_db_instances.py +4 -4
- runbooks/inventory/list_route53_hosted_zones.py +3 -3
- runbooks/inventory/list_servicecatalog_provisioned_products.py +5 -5
- runbooks/inventory/list_sns_topics.py +4 -4
- runbooks/inventory/list_ssm_parameters.py +6 -3
- runbooks/inventory/list_vpc_subnets.py +8 -4
- runbooks/inventory/list_vpcs.py +15 -4
- runbooks/inventory/mcp_vpc_validator.py +6 -0
- runbooks/inventory/organizations_discovery.py +17 -3
- runbooks/inventory/organizations_utils.py +553 -0
- runbooks/inventory/output_formatters.py +422 -0
- runbooks/inventory/recover_cfn_stack_ids.py +5 -5
- runbooks/inventory/run_on_multi_accounts.py +3 -3
- runbooks/inventory/tag_coverage.py +481 -0
- runbooks/inventory/validation_utils.py +358 -0
- runbooks/inventory/verify_ec2_security_groups.py +18 -5
- runbooks/inventory/vpc_architecture_validator.py +7 -1
- runbooks/inventory/vpc_dependency_analyzer.py +6 -0
- runbooks/main_final.py +2 -2
- runbooks/main_ultra_minimal.py +2 -2
- runbooks/mcp/integration.py +6 -4
- runbooks/remediation/acm_remediation.py +2 -2
- runbooks/remediation/cloudtrail_remediation.py +2 -2
- runbooks/remediation/cognito_remediation.py +2 -2
- runbooks/remediation/dynamodb_remediation.py +2 -2
- runbooks/remediation/ec2_remediation.py +2 -2
- runbooks/remediation/kms_remediation.py +2 -2
- runbooks/remediation/lambda_remediation.py +2 -2
- runbooks/remediation/rds_remediation.py +2 -2
- runbooks/remediation/s3_remediation.py +1 -1
- runbooks/vpc/cloudtrail_audit_integration.py +1 -1
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/METADATA +74 -4
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/RECORD +106 -100
- runbooks/__init__.py.backup +0 -134
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/WHEEL +0 -0
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.9.dist-info → runbooks-1.1.10.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,787 @@
|
|
1
|
+
"""
|
2
|
+
Configuration loader with hierarchical precedence for CloudOps Runbooks.
|
3
|
+
|
4
|
+
Precedence (highest to lowest):
|
5
|
+
1. CLI parameters (--tag-mappings '{"wbs_code": "ProjectCode"}')
|
6
|
+
2. Environment variables (RUNBOOKS_TAG_WBS_CODE=ProjectCode)
|
7
|
+
3. Project config file (./.runbooks.yaml)
|
8
|
+
4. User config file (~/.runbooks/config.yaml)
|
9
|
+
5. Default values (AWS best practice tag names)
|
10
|
+
|
11
|
+
This module provides a robust configuration loading system that supports
|
12
|
+
hierarchical configuration sources with intelligent caching and validation.
|
13
|
+
The ConfigLoader class implements the singleton pattern for consistent
|
14
|
+
configuration access across the application.
|
15
|
+
|
16
|
+
Author: CloudOps-Runbooks Enterprise Team
|
17
|
+
Version: 1.1.10
|
18
|
+
"""
|
19
|
+
|
20
|
+
import os
|
21
|
+
import json
|
22
|
+
import logging
|
23
|
+
import re
|
24
|
+
from pathlib import Path
|
25
|
+
from typing import Dict, Any, Optional, Tuple, List
|
26
|
+
from datetime import datetime, timedelta
|
27
|
+
import yaml
|
28
|
+
|
29
|
+
from runbooks.common.config_schema import (
|
30
|
+
TAG_MAPPING_SCHEMA,
|
31
|
+
VALIDATION_RULES,
|
32
|
+
get_allowed_field_names,
|
33
|
+
is_reserved_tag_key,
|
34
|
+
validate_field_name_format,
|
35
|
+
)
|
36
|
+
|
37
|
+
# Configure module logger
|
38
|
+
logger = logging.getLogger(__name__)
|
39
|
+
|
40
|
+
|
41
|
+
# =============================================================================
|
42
|
+
# CONFIGURATION CACHE WITH TTL
|
43
|
+
# =============================================================================
|
44
|
+
|
45
|
+
|
46
|
+
class ConfigCache:
|
47
|
+
"""
|
48
|
+
Thread-safe configuration cache with TTL (Time-To-Live).
|
49
|
+
|
50
|
+
Provides efficient caching of configuration data with automatic expiration
|
51
|
+
to balance performance and freshness. Cache entries automatically expire
|
52
|
+
after the configured TTL period.
|
53
|
+
|
54
|
+
Attributes:
|
55
|
+
default_ttl: Default TTL in seconds (3600 = 1 hour)
|
56
|
+
_cache: Internal cache storage (key -> (value, timestamp))
|
57
|
+
_ttl: Time-to-live duration in seconds
|
58
|
+
|
59
|
+
Example:
|
60
|
+
>>> cache = ConfigCache(ttl_seconds=300)
|
61
|
+
>>> cache.set('config', {'key': 'value'})
|
62
|
+
>>> config = cache.get('config')
|
63
|
+
>>> config is not None
|
64
|
+
True
|
65
|
+
"""
|
66
|
+
|
67
|
+
def __init__(self, ttl_seconds: int = 3600):
|
68
|
+
"""
|
69
|
+
Initialize configuration cache with specified TTL.
|
70
|
+
|
71
|
+
Args:
|
72
|
+
ttl_seconds: Time-to-live in seconds (default: 3600 = 1 hour)
|
73
|
+
"""
|
74
|
+
self._cache: Dict[str, Tuple[Any, datetime]] = {}
|
75
|
+
self._ttl = timedelta(seconds=ttl_seconds)
|
76
|
+
logger.debug(f"ConfigCache initialized with TTL={ttl_seconds}s")
|
77
|
+
|
78
|
+
def get(self, key: str) -> Optional[Any]:
|
79
|
+
"""
|
80
|
+
Get cached value if not expired.
|
81
|
+
|
82
|
+
Args:
|
83
|
+
key: Cache key identifier
|
84
|
+
|
85
|
+
Returns:
|
86
|
+
Cached value if found and not expired, None otherwise
|
87
|
+
|
88
|
+
Example:
|
89
|
+
>>> cache = ConfigCache()
|
90
|
+
>>> cache.set('test', {'data': 'value'})
|
91
|
+
>>> result = cache.get('test')
|
92
|
+
>>> result is not None
|
93
|
+
True
|
94
|
+
"""
|
95
|
+
if key not in self._cache:
|
96
|
+
logger.debug(f"Cache miss: key='{key}'")
|
97
|
+
return None
|
98
|
+
|
99
|
+
value, timestamp = self._cache[key]
|
100
|
+
age = datetime.now() - timestamp
|
101
|
+
|
102
|
+
if age > self._ttl:
|
103
|
+
logger.debug(f"Cache expired: key='{key}', age={age.total_seconds()}s")
|
104
|
+
del self._cache[key]
|
105
|
+
return None
|
106
|
+
|
107
|
+
logger.debug(f"Cache hit: key='{key}', age={age.total_seconds()}s")
|
108
|
+
return value
|
109
|
+
|
110
|
+
def set(self, key: str, value: Any) -> None:
|
111
|
+
"""
|
112
|
+
Set cache value with current timestamp.
|
113
|
+
|
114
|
+
Args:
|
115
|
+
key: Cache key identifier
|
116
|
+
value: Value to cache
|
117
|
+
|
118
|
+
Example:
|
119
|
+
>>> cache = ConfigCache()
|
120
|
+
>>> cache.set('config', {'wbs_code': 'WBS'})
|
121
|
+
"""
|
122
|
+
self._cache[key] = (value, datetime.now())
|
123
|
+
logger.debug(f"Cache set: key='{key}'")
|
124
|
+
|
125
|
+
def clear(self) -> None:
|
126
|
+
"""
|
127
|
+
Clear all cached entries.
|
128
|
+
|
129
|
+
Example:
|
130
|
+
>>> cache = ConfigCache()
|
131
|
+
>>> cache.set('test', 'value')
|
132
|
+
>>> cache.clear()
|
133
|
+
>>> cache.get('test') is None
|
134
|
+
True
|
135
|
+
"""
|
136
|
+
count = len(self._cache)
|
137
|
+
self._cache.clear()
|
138
|
+
logger.info(f"Cache cleared: {count} entries removed")
|
139
|
+
|
140
|
+
|
141
|
+
# =============================================================================
|
142
|
+
# CONFIGURATION LOADER WITH HIERARCHICAL PRECEDENCE
|
143
|
+
# =============================================================================
|
144
|
+
|
145
|
+
|
146
|
+
class ConfigLoader:
|
147
|
+
"""
|
148
|
+
Load configuration from hierarchical sources with precedence.
|
149
|
+
|
150
|
+
Implements a comprehensive configuration loading system that combines
|
151
|
+
multiple configuration sources following a strict precedence order:
|
152
|
+
1. CLI parameters (highest priority)
|
153
|
+
2. Environment variables
|
154
|
+
3. Project config file (./.runbooks.yaml)
|
155
|
+
4. User config file (~/.runbooks/config.yaml)
|
156
|
+
5. Default values (lowest priority)
|
157
|
+
|
158
|
+
The loader uses intelligent caching to optimize repeated configuration
|
159
|
+
access and provides comprehensive validation of all configuration data.
|
160
|
+
|
161
|
+
Attributes:
|
162
|
+
DEFAULT_TAG_MAPPINGS: Default AWS tag mappings (17 fields across 4 tiers)
|
163
|
+
PROJECT_CONFIG_PATHS: Project-level config file search paths
|
164
|
+
USER_CONFIG_PATHS: User-level config file search paths
|
165
|
+
ENV_VAR_PREFIX: Environment variable prefix for tag mappings
|
166
|
+
|
167
|
+
Example:
|
168
|
+
>>> loader = ConfigLoader()
|
169
|
+
>>> mappings = loader.load_tag_mappings()
|
170
|
+
>>> 'wbs_code' in mappings
|
171
|
+
True
|
172
|
+
>>> sources = loader.get_config_sources()
|
173
|
+
>>> 'defaults' in sources
|
174
|
+
True
|
175
|
+
"""
|
176
|
+
|
177
|
+
# Default tag mappings following AWS best practices (17 fields across 4 tiers)
|
178
|
+
DEFAULT_TAG_MAPPINGS: Dict[str, str] = {
|
179
|
+
# TIER 1: Business Metadata (Critical for cost allocation and accountability)
|
180
|
+
"wbs_code": "WBS",
|
181
|
+
"cost_group": "CostGroup",
|
182
|
+
"technical_lead": "TechnicalLead",
|
183
|
+
"account_owner": "AccountOwner",
|
184
|
+
# TIER 2: Governance Metadata (Important for organizational structure)
|
185
|
+
"business_unit": "BusinessUnit",
|
186
|
+
"functional_area": "FunctionalArea",
|
187
|
+
"managed_by": "ManagedBy",
|
188
|
+
"product_owner": "ProductOwner",
|
189
|
+
# TIER 3: Operational Metadata (Standard operational requirements)
|
190
|
+
"purpose": "Purpose",
|
191
|
+
"environment": "Environment",
|
192
|
+
"compliance_scope": "ComplianceScope",
|
193
|
+
"data_classification": "DataClassification",
|
194
|
+
# TIER 4: Extended Metadata (Optional supplementary information)
|
195
|
+
"project_name": "ProjectName",
|
196
|
+
"budget_code": "BudgetCode",
|
197
|
+
"support_tier": "SupportTier",
|
198
|
+
"created_date": "CreatedDate",
|
199
|
+
"expiry_date": "ExpiryDate",
|
200
|
+
}
|
201
|
+
|
202
|
+
# Configuration file search paths (in priority order)
|
203
|
+
PROJECT_CONFIG_PATHS: List[str] = [
|
204
|
+
".runbooks.yaml", # Current directory (highest priority)
|
205
|
+
".runbooks.yml", # Alternative extension
|
206
|
+
]
|
207
|
+
|
208
|
+
USER_CONFIG_PATHS: List[str] = [
|
209
|
+
"~/.runbooks/config.yaml", # User home directory
|
210
|
+
"~/.runbooks/config.yml", # Alternative extension
|
211
|
+
"~/.config/runbooks/config.yaml", # XDG config directory
|
212
|
+
"~/.config/runbooks/config.yml", # Alternative extension
|
213
|
+
]
|
214
|
+
|
215
|
+
# Environment variable configuration
|
216
|
+
ENV_VAR_PREFIX: str = "RUNBOOKS_TAG_"
|
217
|
+
|
218
|
+
def __init__(self, cache_ttl: int = 3600):
|
219
|
+
"""
|
220
|
+
Initialize ConfigLoader with configuration cache.
|
221
|
+
|
222
|
+
Args:
|
223
|
+
cache_ttl: Cache time-to-live in seconds (default: 3600 = 1 hour)
|
224
|
+
|
225
|
+
Example:
|
226
|
+
>>> loader = ConfigLoader(cache_ttl=1800)
|
227
|
+
>>> isinstance(loader._cache, ConfigCache)
|
228
|
+
True
|
229
|
+
"""
|
230
|
+
self._cache = ConfigCache(ttl_seconds=cache_ttl)
|
231
|
+
self._config_sources: List[str] = []
|
232
|
+
logger.info(f"ConfigLoader initialized with cache_ttl={cache_ttl}s")
|
233
|
+
|
234
|
+
def load_tag_mappings(
|
235
|
+
self,
|
236
|
+
cli_overrides: Optional[Dict[str, str]] = None,
|
237
|
+
force_reload: bool = False,
|
238
|
+
) -> Dict[str, str]:
|
239
|
+
"""
|
240
|
+
Load tag mappings with hierarchical precedence.
|
241
|
+
|
242
|
+
Combines configuration from multiple sources following strict precedence:
|
243
|
+
1. CLI overrides (highest priority)
|
244
|
+
2. Environment variables (RUNBOOKS_TAG_*)
|
245
|
+
3. Project config file (./.runbooks.yaml)
|
246
|
+
4. User config file (~/.runbooks/config.yaml)
|
247
|
+
5. Default values (lowest priority)
|
248
|
+
|
249
|
+
Higher priority sources override lower priority sources for individual
|
250
|
+
tag mappings while preserving other mappings from lower priority sources.
|
251
|
+
|
252
|
+
Args:
|
253
|
+
cli_overrides: Optional CLI parameter overrides (highest priority)
|
254
|
+
force_reload: Force reload from sources, bypassing cache
|
255
|
+
|
256
|
+
Returns:
|
257
|
+
Merged tag mappings dictionary with all sources combined
|
258
|
+
|
259
|
+
Raises:
|
260
|
+
ValueError: If configuration validation fails
|
261
|
+
|
262
|
+
Example:
|
263
|
+
>>> loader = ConfigLoader()
|
264
|
+
>>> mappings = loader.load_tag_mappings()
|
265
|
+
>>> len(mappings) == 17 # Default has 17 mappings
|
266
|
+
True
|
267
|
+
>>> overrides = {'wbs_code': 'ProjectCode'}
|
268
|
+
>>> mappings = loader.load_tag_mappings(cli_overrides=overrides)
|
269
|
+
>>> mappings['wbs_code']
|
270
|
+
'ProjectCode'
|
271
|
+
"""
|
272
|
+
# Check cache if not forcing reload
|
273
|
+
if not force_reload:
|
274
|
+
cache_key = self._get_cache_key(cli_overrides)
|
275
|
+
cached = self._cache.get(cache_key)
|
276
|
+
if cached is not None:
|
277
|
+
logger.debug(f"Returning cached tag mappings: {len(cached)} entries")
|
278
|
+
return cached
|
279
|
+
|
280
|
+
logger.info("Loading tag mappings from hierarchical sources")
|
281
|
+
self._config_sources = []
|
282
|
+
|
283
|
+
# Layer 1: Start with defaults (lowest priority)
|
284
|
+
result = self._get_defaults()
|
285
|
+
self._config_sources.append("defaults")
|
286
|
+
logger.debug(f"Layer 1 (defaults): {len(result)} mappings")
|
287
|
+
|
288
|
+
# Layer 2: Merge user config file
|
289
|
+
user_config = self._load_from_config_file(self.USER_CONFIG_PATHS)
|
290
|
+
if user_config:
|
291
|
+
result.update(user_config)
|
292
|
+
logger.debug(f"Layer 2 (user config): merged {len(user_config)} mappings")
|
293
|
+
|
294
|
+
# Layer 3: Merge project config file (higher priority than user config)
|
295
|
+
project_config = self._load_from_config_file(self.PROJECT_CONFIG_PATHS)
|
296
|
+
if project_config:
|
297
|
+
result.update(project_config)
|
298
|
+
logger.debug(f"Layer 3 (project config): merged {len(project_config)} mappings")
|
299
|
+
|
300
|
+
# Layer 4: Merge environment variables
|
301
|
+
env_vars = self._load_from_env_vars()
|
302
|
+
if env_vars:
|
303
|
+
result.update(env_vars)
|
304
|
+
logger.debug(f"Layer 4 (env vars): merged {len(env_vars)} mappings")
|
305
|
+
|
306
|
+
# Layer 5: Merge CLI overrides (highest priority)
|
307
|
+
if cli_overrides:
|
308
|
+
# Validate CLI overrides
|
309
|
+
is_valid, errors = self._validate_tag_mappings(cli_overrides)
|
310
|
+
if not is_valid:
|
311
|
+
error_msg = f"Invalid CLI overrides: {'; '.join(errors)}"
|
312
|
+
logger.error(error_msg)
|
313
|
+
raise ValueError(error_msg)
|
314
|
+
|
315
|
+
result.update(cli_overrides)
|
316
|
+
self._config_sources.append("cli_overrides")
|
317
|
+
logger.debug(f"Layer 5 (CLI overrides): merged {len(cli_overrides)} mappings")
|
318
|
+
|
319
|
+
# Final validation
|
320
|
+
is_valid, errors = self._validate_tag_mappings(result)
|
321
|
+
if not is_valid:
|
322
|
+
error_msg = f"Configuration validation failed: {'; '.join(errors)}"
|
323
|
+
logger.error(error_msg)
|
324
|
+
raise ValueError(error_msg)
|
325
|
+
|
326
|
+
# Cache result
|
327
|
+
cache_key = self._get_cache_key(cli_overrides)
|
328
|
+
self._cache.set(cache_key, result)
|
329
|
+
|
330
|
+
logger.info(
|
331
|
+
f"Tag mappings loaded successfully: {len(result)} mappings from "
|
332
|
+
f"{len(self._config_sources)} sources ({', '.join(self._config_sources)})"
|
333
|
+
)
|
334
|
+
|
335
|
+
return result
|
336
|
+
|
337
|
+
def _get_defaults(self) -> Dict[str, str]:
|
338
|
+
"""
|
339
|
+
Get default tag mappings.
|
340
|
+
|
341
|
+
Returns a copy of the default tag mappings to prevent accidental
|
342
|
+
modification of the class constant.
|
343
|
+
|
344
|
+
Returns:
|
345
|
+
Copy of DEFAULT_TAG_MAPPINGS dictionary
|
346
|
+
|
347
|
+
Example:
|
348
|
+
>>> loader = ConfigLoader()
|
349
|
+
>>> defaults = loader._get_defaults()
|
350
|
+
>>> len(defaults) == 17
|
351
|
+
True
|
352
|
+
>>> defaults['wbs_code']
|
353
|
+
'WBS'
|
354
|
+
"""
|
355
|
+
return self.DEFAULT_TAG_MAPPINGS.copy()
|
356
|
+
|
357
|
+
def _load_from_config_file(self, search_paths: List[str]) -> Optional[Dict[str, str]]:
|
358
|
+
"""
|
359
|
+
Load tag mappings from YAML configuration file.
|
360
|
+
|
361
|
+
Searches for configuration files in the provided paths (in order)
|
362
|
+
and loads the first valid file found. Configuration files must follow
|
363
|
+
the expected structure: runbooks.inventory.tag_mappings
|
364
|
+
|
365
|
+
Args:
|
366
|
+
search_paths: List of file paths to search (in priority order)
|
367
|
+
|
368
|
+
Returns:
|
369
|
+
Tag mappings from config file, or None if no valid file found
|
370
|
+
|
371
|
+
Example:
|
372
|
+
>>> loader = ConfigLoader()
|
373
|
+
>>> # Assumes ~/.runbooks/config.yaml exists with valid config
|
374
|
+
>>> mappings = loader._load_from_config_file(loader.USER_CONFIG_PATHS)
|
375
|
+
>>> mappings is None or isinstance(mappings, dict)
|
376
|
+
True
|
377
|
+
"""
|
378
|
+
for path_str in search_paths:
|
379
|
+
# Expand user home directory
|
380
|
+
path = Path(path_str).expanduser()
|
381
|
+
|
382
|
+
if not path.exists():
|
383
|
+
continue
|
384
|
+
|
385
|
+
logger.debug(f"Found config file: {path}")
|
386
|
+
|
387
|
+
try:
|
388
|
+
# Load YAML file
|
389
|
+
with open(path, "r") as f:
|
390
|
+
config_data = yaml.safe_load(f)
|
391
|
+
|
392
|
+
if not config_data:
|
393
|
+
logger.warning(f"Empty config file: {path}")
|
394
|
+
continue
|
395
|
+
|
396
|
+
# Extract tag mappings from nested structure
|
397
|
+
tag_mappings = self._extract_tag_mappings(config_data)
|
398
|
+
|
399
|
+
if not tag_mappings:
|
400
|
+
logger.warning(f"No tag_mappings found in config file: {path}")
|
401
|
+
continue
|
402
|
+
|
403
|
+
# Validate configuration structure
|
404
|
+
is_valid, errors = self.validate_config(config_data)
|
405
|
+
if not is_valid:
|
406
|
+
logger.warning(f"Invalid config file structure: {path} - {'; '.join(errors)}")
|
407
|
+
continue
|
408
|
+
|
409
|
+
# Validate individual tag mappings
|
410
|
+
is_valid, errors = self._validate_tag_mappings(tag_mappings)
|
411
|
+
if not is_valid:
|
412
|
+
logger.warning(f"Invalid tag mappings in config file: {path} - {'; '.join(errors)}")
|
413
|
+
continue
|
414
|
+
|
415
|
+
# Record successful load
|
416
|
+
source_type = "project_config" if path_str in self.PROJECT_CONFIG_PATHS else "user_config"
|
417
|
+
self._config_sources.append(f"{source_type}:{path}")
|
418
|
+
|
419
|
+
logger.info(f"Loaded tag mappings from config file: {path} ({len(tag_mappings)} mappings)")
|
420
|
+
return tag_mappings
|
421
|
+
|
422
|
+
except yaml.YAMLError as e:
|
423
|
+
logger.warning(f"YAML parse error in config file: {path} - {e}")
|
424
|
+
continue
|
425
|
+
|
426
|
+
except Exception as e:
|
427
|
+
logger.warning(f"Error loading config file: {path} - {e}")
|
428
|
+
continue
|
429
|
+
|
430
|
+
logger.debug(f"No valid config file found in search paths: {search_paths}")
|
431
|
+
return None
|
432
|
+
|
433
|
+
def _extract_tag_mappings(self, config_data: Dict[str, Any]) -> Optional[Dict[str, str]]:
|
434
|
+
"""
|
435
|
+
Extract tag_mappings from nested configuration structure.
|
436
|
+
|
437
|
+
Expected structure:
|
438
|
+
{
|
439
|
+
"runbooks": {
|
440
|
+
"inventory": {
|
441
|
+
"tag_mappings": {
|
442
|
+
"wbs_code": "WBS",
|
443
|
+
...
|
444
|
+
}
|
445
|
+
}
|
446
|
+
}
|
447
|
+
}
|
448
|
+
|
449
|
+
Args:
|
450
|
+
config_data: Loaded YAML configuration data
|
451
|
+
|
452
|
+
Returns:
|
453
|
+
Tag mappings dictionary, or None if not found
|
454
|
+
|
455
|
+
Example:
|
456
|
+
>>> loader = ConfigLoader()
|
457
|
+
>>> config = {
|
458
|
+
... 'runbooks': {
|
459
|
+
... 'inventory': {
|
460
|
+
... 'tag_mappings': {'wbs_code': 'WBS'}
|
461
|
+
... }
|
462
|
+
... }
|
463
|
+
... }
|
464
|
+
>>> mappings = loader._extract_tag_mappings(config)
|
465
|
+
>>> mappings['wbs_code']
|
466
|
+
'WBS'
|
467
|
+
"""
|
468
|
+
try:
|
469
|
+
return config_data.get("runbooks", {}).get("inventory", {}).get("tag_mappings")
|
470
|
+
except (AttributeError, KeyError):
|
471
|
+
return None
|
472
|
+
|
473
|
+
def _load_from_env_vars(self) -> Dict[str, str]:
|
474
|
+
"""
|
475
|
+
Load tag mappings from environment variables.
|
476
|
+
|
477
|
+
Environment variables follow the pattern:
|
478
|
+
RUNBOOKS_TAG_<FIELD_NAME_UPPER>=<TAG_KEY>
|
479
|
+
|
480
|
+
Example:
|
481
|
+
- RUNBOOKS_TAG_WBS_CODE=ProjectCode → {'wbs_code': 'ProjectCode'}
|
482
|
+
- RUNBOOKS_TAG_ENVIRONMENT=Env → {'environment': 'Env'}
|
483
|
+
|
484
|
+
Returns:
|
485
|
+
Dictionary of tag mappings from environment variables
|
486
|
+
|
487
|
+
Example:
|
488
|
+
>>> import os
|
489
|
+
>>> os.environ['RUNBOOKS_TAG_WBS_CODE'] = 'ProjectCode'
|
490
|
+
>>> loader = ConfigLoader()
|
491
|
+
>>> env_mappings = loader._load_from_env_vars()
|
492
|
+
>>> env_mappings.get('wbs_code')
|
493
|
+
'ProjectCode'
|
494
|
+
"""
|
495
|
+
result = {}
|
496
|
+
allowed_fields = set(get_allowed_field_names())
|
497
|
+
|
498
|
+
for env_var, tag_key in os.environ.items():
|
499
|
+
# Check if this is a runbooks tag mapping env var
|
500
|
+
if not env_var.startswith(self.ENV_VAR_PREFIX):
|
501
|
+
continue
|
502
|
+
|
503
|
+
# Extract field name from env var (convert UPPER_CASE to lower_case)
|
504
|
+
field_name_upper = env_var[len(self.ENV_VAR_PREFIX) :]
|
505
|
+
field_name = field_name_upper.lower()
|
506
|
+
|
507
|
+
# Validate field name
|
508
|
+
if field_name not in allowed_fields:
|
509
|
+
logger.warning(
|
510
|
+
f"Ignoring unknown field in env var '{env_var}': '{field_name}' "
|
511
|
+
f"(allowed fields: {', '.join(sorted(allowed_fields))})"
|
512
|
+
)
|
513
|
+
continue
|
514
|
+
|
515
|
+
# Validate tag key
|
516
|
+
if not tag_key or len(tag_key) > 128:
|
517
|
+
logger.warning(
|
518
|
+
f"Ignoring invalid tag key in env var '{env_var}': '{tag_key}' "
|
519
|
+
f"(must be 1-128 characters)"
|
520
|
+
)
|
521
|
+
continue
|
522
|
+
|
523
|
+
if is_reserved_tag_key(tag_key):
|
524
|
+
logger.warning(
|
525
|
+
f"Ignoring reserved tag key in env var '{env_var}': '{tag_key}' "
|
526
|
+
f"(reserved keys: Name, aws:*)"
|
527
|
+
)
|
528
|
+
continue
|
529
|
+
|
530
|
+
result[field_name] = tag_key
|
531
|
+
logger.debug(f"Loaded from env var '{env_var}': {field_name}='{tag_key}'")
|
532
|
+
|
533
|
+
if result:
|
534
|
+
self._config_sources.append(f"env_vars:{len(result)}_mappings")
|
535
|
+
logger.info(f"Loaded {len(result)} tag mappings from environment variables")
|
536
|
+
|
537
|
+
return result
|
538
|
+
|
539
|
+
def _validate_tag_mappings(self, tag_mappings: Dict[str, str]) -> Tuple[bool, List[str]]:
|
540
|
+
"""
|
541
|
+
Validate individual tag mappings.
|
542
|
+
|
543
|
+
Checks:
|
544
|
+
1. Field names are in allowed list
|
545
|
+
2. Field names follow format rules (lowercase + underscores)
|
546
|
+
3. Tag keys are valid (1-128 chars, not reserved)
|
547
|
+
|
548
|
+
Args:
|
549
|
+
tag_mappings: Dictionary of field_name -> tag_key mappings
|
550
|
+
|
551
|
+
Returns:
|
552
|
+
Tuple of (is_valid, error_messages)
|
553
|
+
|
554
|
+
Example:
|
555
|
+
>>> loader = ConfigLoader()
|
556
|
+
>>> valid_mappings = {'wbs_code': 'WBS'}
|
557
|
+
>>> is_valid, errors = loader._validate_tag_mappings(valid_mappings)
|
558
|
+
>>> is_valid
|
559
|
+
True
|
560
|
+
>>> invalid_mappings = {'INVALID': 'aws:reserved'}
|
561
|
+
>>> is_valid, errors = loader._validate_tag_mappings(invalid_mappings)
|
562
|
+
>>> is_valid
|
563
|
+
False
|
564
|
+
"""
|
565
|
+
errors = []
|
566
|
+
allowed_fields = set(get_allowed_field_names())
|
567
|
+
|
568
|
+
for field_name, tag_key in tag_mappings.items():
|
569
|
+
# Validate field name format
|
570
|
+
if not validate_field_name_format(field_name):
|
571
|
+
errors.append(
|
572
|
+
f"Field name '{field_name}' has invalid format "
|
573
|
+
f"(must be lowercase with underscores: ^[a-z_]+$)"
|
574
|
+
)
|
575
|
+
continue
|
576
|
+
|
577
|
+
# Validate field name is allowed
|
578
|
+
if field_name not in allowed_fields:
|
579
|
+
errors.append(
|
580
|
+
f"Unknown field name '{field_name}' "
|
581
|
+
f"(allowed: {', '.join(sorted(allowed_fields))})"
|
582
|
+
)
|
583
|
+
continue
|
584
|
+
|
585
|
+
# Validate tag key length
|
586
|
+
if not tag_key or len(tag_key) > 128:
|
587
|
+
errors.append(f"Tag key for '{field_name}' has invalid length: '{tag_key}' (must be 1-128 chars)")
|
588
|
+
continue
|
589
|
+
|
590
|
+
# Validate tag key is not reserved
|
591
|
+
if is_reserved_tag_key(tag_key):
|
592
|
+
errors.append(
|
593
|
+
f"Tag key for '{field_name}' is reserved: '{tag_key}' (reserved: Name, aws:*)"
|
594
|
+
)
|
595
|
+
continue
|
596
|
+
|
597
|
+
is_valid = len(errors) == 0
|
598
|
+
return is_valid, errors
|
599
|
+
|
600
|
+
def validate_config(self, config_data: Dict[str, Any]) -> Tuple[bool, List[str]]:
|
601
|
+
"""
|
602
|
+
Validate configuration file structure.
|
603
|
+
|
604
|
+
Validates the complete configuration structure against the schema,
|
605
|
+
including version format, nested structure, and tag mappings.
|
606
|
+
|
607
|
+
Args:
|
608
|
+
config_data: Complete configuration data dictionary
|
609
|
+
|
610
|
+
Returns:
|
611
|
+
Tuple of (is_valid, error_messages)
|
612
|
+
|
613
|
+
Example:
|
614
|
+
>>> loader = ConfigLoader()
|
615
|
+
>>> valid_config = {
|
616
|
+
... 'runbooks': {
|
617
|
+
... 'version': '1.1.10',
|
618
|
+
... 'inventory': {
|
619
|
+
... 'tag_mappings': {'wbs_code': 'WBS'}
|
620
|
+
... }
|
621
|
+
... }
|
622
|
+
... }
|
623
|
+
>>> is_valid, errors = loader.validate_config(valid_config)
|
624
|
+
>>> is_valid
|
625
|
+
True
|
626
|
+
"""
|
627
|
+
errors = []
|
628
|
+
|
629
|
+
# Check top-level structure
|
630
|
+
if not isinstance(config_data, dict):
|
631
|
+
errors.append("Config must be a dictionary")
|
632
|
+
return False, errors
|
633
|
+
|
634
|
+
if "runbooks" not in config_data:
|
635
|
+
errors.append("Config must have 'runbooks' key")
|
636
|
+
return False, errors
|
637
|
+
|
638
|
+
runbooks_config = config_data["runbooks"]
|
639
|
+
|
640
|
+
if not isinstance(runbooks_config, dict):
|
641
|
+
errors.append("'runbooks' must be a dictionary")
|
642
|
+
return False, errors
|
643
|
+
|
644
|
+
# Validate version if present
|
645
|
+
if "version" in runbooks_config:
|
646
|
+
version = runbooks_config["version"]
|
647
|
+
if not isinstance(version, str):
|
648
|
+
errors.append(f"'runbooks.version' must be a string, got {type(version).__name__}")
|
649
|
+
elif not re.match(r"^\d+\.\d+\.\d+$", version):
|
650
|
+
errors.append(
|
651
|
+
f"'runbooks.version' must follow semantic versioning (e.g., '1.1.10'), got '{version}'"
|
652
|
+
)
|
653
|
+
|
654
|
+
# Check inventory structure
|
655
|
+
if "inventory" not in runbooks_config:
|
656
|
+
errors.append("Config must have 'runbooks.inventory' key")
|
657
|
+
return False, errors
|
658
|
+
|
659
|
+
inventory_config = runbooks_config["inventory"]
|
660
|
+
|
661
|
+
if not isinstance(inventory_config, dict):
|
662
|
+
errors.append("'runbooks.inventory' must be a dictionary")
|
663
|
+
return False, errors
|
664
|
+
|
665
|
+
# Check tag_mappings structure
|
666
|
+
if "tag_mappings" not in inventory_config:
|
667
|
+
errors.append("Config must have 'runbooks.inventory.tag_mappings' key")
|
668
|
+
return False, errors
|
669
|
+
|
670
|
+
tag_mappings = inventory_config["tag_mappings"]
|
671
|
+
|
672
|
+
if not isinstance(tag_mappings, dict):
|
673
|
+
errors.append("'runbooks.inventory.tag_mappings' must be a dictionary")
|
674
|
+
return False, errors
|
675
|
+
|
676
|
+
# Validate individual tag mappings
|
677
|
+
tag_validation_ok, tag_errors = self._validate_tag_mappings(tag_mappings)
|
678
|
+
if not tag_validation_ok:
|
679
|
+
errors.extend(tag_errors)
|
680
|
+
|
681
|
+
is_valid = len(errors) == 0
|
682
|
+
return is_valid, errors
|
683
|
+
|
684
|
+
def get_config_sources(self) -> List[str]:
|
685
|
+
"""
|
686
|
+
Get list of configuration sources used in last load.
|
687
|
+
|
688
|
+
Returns:
|
689
|
+
List of source identifiers (e.g., 'defaults', 'user_config:/path', 'env_vars')
|
690
|
+
|
691
|
+
Example:
|
692
|
+
>>> loader = ConfigLoader()
|
693
|
+
>>> loader.load_tag_mappings() # doctest: +SKIP
|
694
|
+
>>> sources = loader.get_config_sources()
|
695
|
+
>>> 'defaults' in sources
|
696
|
+
True
|
697
|
+
"""
|
698
|
+
return self._config_sources.copy()
|
699
|
+
|
700
|
+
def clear_cache(self) -> None:
|
701
|
+
"""
|
702
|
+
Clear configuration cache.
|
703
|
+
|
704
|
+
Forces reload of configuration from sources on next load_tag_mappings() call.
|
705
|
+
|
706
|
+
Example:
|
707
|
+
>>> loader = ConfigLoader()
|
708
|
+
>>> loader.load_tag_mappings() # doctest: +SKIP
|
709
|
+
>>> loader.clear_cache()
|
710
|
+
"""
|
711
|
+
self._cache.clear()
|
712
|
+
logger.info("Configuration cache cleared")
|
713
|
+
|
714
|
+
def _get_cache_key(self, cli_overrides: Optional[Dict[str, str]]) -> str:
|
715
|
+
"""
|
716
|
+
Generate cache key based on CLI overrides.
|
717
|
+
|
718
|
+
Different CLI overrides result in different cache keys to ensure
|
719
|
+
correct cached configuration retrieval.
|
720
|
+
|
721
|
+
Args:
|
722
|
+
cli_overrides: Optional CLI parameter overrides
|
723
|
+
|
724
|
+
Returns:
|
725
|
+
Cache key string
|
726
|
+
|
727
|
+
Example:
|
728
|
+
>>> loader = ConfigLoader()
|
729
|
+
>>> key1 = loader._get_cache_key(None)
|
730
|
+
>>> key2 = loader._get_cache_key({'wbs_code': 'WBS'})
|
731
|
+
>>> key1 != key2
|
732
|
+
True
|
733
|
+
"""
|
734
|
+
if cli_overrides:
|
735
|
+
# Create deterministic key from sorted overrides
|
736
|
+
sorted_overrides = json.dumps(cli_overrides, sort_keys=True)
|
737
|
+
return f"tag_mappings:{sorted_overrides}"
|
738
|
+
return "tag_mappings:default"
|
739
|
+
|
740
|
+
|
741
|
+
# =============================================================================
|
742
|
+
# SINGLETON INSTANCE MANAGEMENT
|
743
|
+
# =============================================================================
|
744
|
+
|
745
|
+
# Global singleton instance
|
746
|
+
_config_loader_instance: Optional[ConfigLoader] = None
|
747
|
+
|
748
|
+
|
749
|
+
def get_config_loader(cache_ttl: int = 3600) -> ConfigLoader:
|
750
|
+
"""
|
751
|
+
Get singleton ConfigLoader instance.
|
752
|
+
|
753
|
+
Provides global access to a single ConfigLoader instance throughout
|
754
|
+
the application lifecycle. This ensures consistent configuration
|
755
|
+
access and efficient cache utilization.
|
756
|
+
|
757
|
+
Args:
|
758
|
+
cache_ttl: Cache TTL in seconds (default: 3600 = 1 hour)
|
759
|
+
Only used when creating new instance
|
760
|
+
|
761
|
+
Returns:
|
762
|
+
Singleton ConfigLoader instance
|
763
|
+
|
764
|
+
Example:
|
765
|
+
>>> loader1 = get_config_loader()
|
766
|
+
>>> loader2 = get_config_loader()
|
767
|
+
>>> loader1 is loader2
|
768
|
+
True
|
769
|
+
"""
|
770
|
+
global _config_loader_instance
|
771
|
+
|
772
|
+
if _config_loader_instance is None:
|
773
|
+
_config_loader_instance = ConfigLoader(cache_ttl=cache_ttl)
|
774
|
+
logger.info("Created singleton ConfigLoader instance")
|
775
|
+
|
776
|
+
return _config_loader_instance
|
777
|
+
|
778
|
+
|
779
|
+
# =============================================================================
|
780
|
+
# MODULE EXPORTS
|
781
|
+
# =============================================================================
|
782
|
+
|
783
|
+
__all__ = [
|
784
|
+
"ConfigCache",
|
785
|
+
"ConfigLoader",
|
786
|
+
"get_config_loader",
|
787
|
+
]
|