runbooks 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/cfat/assessment/collectors.py +3 -2
- runbooks/cloudops/cost_optimizer.py +235 -83
- runbooks/cloudops/models.py +8 -2
- runbooks/common/aws_pricing.py +12 -0
- runbooks/common/business_logic.py +1 -1
- runbooks/common/profile_utils.py +213 -310
- runbooks/common/rich_utils.py +15 -21
- runbooks/finops/README.md +3 -3
- runbooks/finops/__init__.py +13 -5
- runbooks/finops/business_case_config.py +5 -5
- runbooks/finops/cli.py +170 -95
- runbooks/finops/cost_optimizer.py +2 -1
- runbooks/finops/cost_processor.py +69 -22
- runbooks/finops/dashboard_router.py +3 -3
- runbooks/finops/dashboard_runner.py +3 -4
- runbooks/finops/embedded_mcp_validator.py +101 -23
- runbooks/finops/enhanced_progress.py +213 -0
- runbooks/finops/finops_scenarios.py +90 -16
- runbooks/finops/markdown_exporter.py +4 -2
- runbooks/finops/multi_dashboard.py +1 -1
- runbooks/finops/nat_gateway_optimizer.py +85 -57
- runbooks/finops/rds_snapshot_optimizer.py +1389 -0
- runbooks/finops/scenario_cli_integration.py +212 -22
- runbooks/finops/scenarios.py +41 -25
- runbooks/finops/single_dashboard.py +68 -9
- runbooks/finops/tests/run_tests.py +5 -3
- runbooks/finops/vpc_cleanup_optimizer.py +1 -1
- runbooks/finops/workspaces_analyzer.py +40 -16
- runbooks/inventory/list_rds_snapshots_aggregator.py +745 -0
- runbooks/main.py +393 -61
- runbooks/operate/executive_dashboard.py +4 -3
- runbooks/remediation/rds_snapshot_list.py +13 -0
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/METADATA +234 -40
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/RECORD +39 -37
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/WHEEL +0 -0
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/entry_points.txt +0 -0
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.1.1.dist-info → runbooks-1.1.3.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,745 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
"""
|
3
|
+
Enhanced RDS Snapshot Discovery via AWS Config Organization Aggregator
|
4
|
+
|
5
|
+
Root Cause Solution: AWS Config organization-aggregator provides cross-account access to RDS snapshots
|
6
|
+
where direct RDS API calls fail due to permissions. This module fixes the infrastructure to discover
|
7
|
+
all enterprise resources across 65 accounts using the proven Config aggregator approach.
|
8
|
+
|
9
|
+
Gap Analysis Results:
|
10
|
+
- AWS Config aggregator in ap-southeast-2 with 'organization-aggregator'
|
11
|
+
- Uses MANAGEMENT_PROFILE successfully for cross-account access
|
12
|
+
- Found 42 snapshots from target account 142964829704 (proves access works)
|
13
|
+
- Gap: Need multi-region support and remove query limits for full 303 snapshots
|
14
|
+
|
15
|
+
Business Case: Enhanced RDS snapshot lifecycle management with comprehensive cross-account visibility
|
16
|
+
Target Coverage: 65 AWS accounts with enterprise resource discovery capabilities
|
17
|
+
Strategic Value: Infrastructure governance and cost optimization through automated snapshot analysis
|
18
|
+
"""
|
19
|
+
|
20
|
+
import logging
|
21
|
+
import json
|
22
|
+
from datetime import datetime, timedelta, timezone
|
23
|
+
from typing import Dict, List, Optional, Tuple, Set
|
24
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
25
|
+
import time
|
26
|
+
|
27
|
+
import click
|
28
|
+
import boto3
|
29
|
+
from botocore.exceptions import ClientError, NoCredentialsError
|
30
|
+
|
31
|
+
from ..common.rich_utils import (
|
32
|
+
console, print_header, print_success, print_error, print_warning, print_info,
|
33
|
+
create_table, create_progress_bar, format_cost
|
34
|
+
)
|
35
|
+
from ..common.profile_utils import get_profile_for_operation
|
36
|
+
|
37
|
+
logger = logging.getLogger(__name__)
|
38
|
+
|
39
|
+
|
40
|
+
class RDSSnapshotConfigAggregator:
|
41
|
+
"""
|
42
|
+
Enhanced RDS Snapshot Discovery using AWS Config Organization Aggregator
|
43
|
+
|
44
|
+
Solves cross-account access limitations by leveraging AWS Config's organization-aggregator
|
45
|
+
to discover RDS snapshots across all accounts where direct RDS API access fails.
|
46
|
+
"""
|
47
|
+
|
48
|
+
def __init__(self, management_profile: str = None, max_workers: int = 10):
|
49
|
+
"""
|
50
|
+
Initialize RDS snapshot discovery with Config aggregator integration
|
51
|
+
|
52
|
+
Args:
|
53
|
+
management_profile: AWS profile with Config aggregator access
|
54
|
+
max_workers: Maximum concurrent workers for multi-region discovery
|
55
|
+
"""
|
56
|
+
self.management_profile = management_profile
|
57
|
+
self.max_workers = max_workers
|
58
|
+
|
59
|
+
# Config aggregator regions for comprehensive coverage
|
60
|
+
self.config_regions = [
|
61
|
+
'us-east-1', 'us-west-2', 'eu-west-1', 'ap-southeast-2',
|
62
|
+
'ap-northeast-1', 'ca-central-1', 'eu-central-1'
|
63
|
+
]
|
64
|
+
|
65
|
+
# Session management
|
66
|
+
self.session = None
|
67
|
+
self.discovered_aggregators = {}
|
68
|
+
self.snapshot_inventory = []
|
69
|
+
|
70
|
+
# Performance metrics
|
71
|
+
self.metrics = {
|
72
|
+
'aggregators_found': 0,
|
73
|
+
'snapshots_discovered': 0,
|
74
|
+
'accounts_covered': 0,
|
75
|
+
'regions_scanned': 0,
|
76
|
+
'api_calls_made': 0,
|
77
|
+
'errors_encountered': 0
|
78
|
+
}
|
79
|
+
|
80
|
+
def initialize_session(self) -> bool:
|
81
|
+
"""Initialize AWS session with management profile"""
|
82
|
+
try:
|
83
|
+
profile = get_profile_for_operation("management", self.management_profile)
|
84
|
+
self.session = boto3.Session(profile_name=profile)
|
85
|
+
|
86
|
+
# Verify access with STS
|
87
|
+
sts_client = self.session.client('sts')
|
88
|
+
identity = sts_client.get_caller_identity()
|
89
|
+
|
90
|
+
print_success(f"✅ Initialized session with profile: {profile}")
|
91
|
+
console.print(f"[dim]Account: {identity['Account']} | ARN: {identity['Arn']}[/dim]")
|
92
|
+
|
93
|
+
return True
|
94
|
+
|
95
|
+
except Exception as e:
|
96
|
+
print_error(f"Failed to initialize session: {e}")
|
97
|
+
return False
|
98
|
+
|
99
|
+
def discover_config_aggregators(self) -> Dict[str, List[str]]:
|
100
|
+
"""
|
101
|
+
Discover AWS Config aggregators across all regions
|
102
|
+
|
103
|
+
Returns:
|
104
|
+
Dict mapping regions to list of aggregator names
|
105
|
+
"""
|
106
|
+
print_info("🔍 Discovering AWS Config aggregators across regions...")
|
107
|
+
|
108
|
+
aggregator_map = {}
|
109
|
+
|
110
|
+
with create_progress_bar() as progress:
|
111
|
+
task_id = progress.add_task(
|
112
|
+
"Scanning regions for Config aggregators...",
|
113
|
+
total=len(self.config_regions)
|
114
|
+
)
|
115
|
+
|
116
|
+
with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
|
117
|
+
future_to_region = {
|
118
|
+
executor.submit(self._find_aggregators_in_region, region): region
|
119
|
+
for region in self.config_regions
|
120
|
+
}
|
121
|
+
|
122
|
+
for future in as_completed(future_to_region):
|
123
|
+
region = future_to_region[future]
|
124
|
+
try:
|
125
|
+
aggregators = future.result()
|
126
|
+
if aggregators:
|
127
|
+
aggregator_map[region] = aggregators
|
128
|
+
self.metrics['aggregators_found'] += len(aggregators)
|
129
|
+
console.print(f"[green]✓[/green] {region}: Found {len(aggregators)} aggregator(s)")
|
130
|
+
else:
|
131
|
+
console.print(f"[dim]○[/dim] {region}: No aggregators found")
|
132
|
+
|
133
|
+
self.metrics['regions_scanned'] += 1
|
134
|
+
|
135
|
+
except Exception as e:
|
136
|
+
print_warning(f"❌ {region}: {str(e)}")
|
137
|
+
self.metrics['errors_encountered'] += 1
|
138
|
+
|
139
|
+
progress.advance(task_id)
|
140
|
+
|
141
|
+
self.discovered_aggregators = aggregator_map
|
142
|
+
|
143
|
+
# Summary
|
144
|
+
total_aggregators = sum(len(aggs) for aggs in aggregator_map.values())
|
145
|
+
if total_aggregators > 0:
|
146
|
+
print_success(f"✅ Found {total_aggregators} Config aggregators across {len(aggregator_map)} regions")
|
147
|
+
else:
|
148
|
+
print_warning("⚠️ No Config aggregators found - RDS snapshot discovery will be limited")
|
149
|
+
|
150
|
+
return aggregator_map
|
151
|
+
|
152
|
+
def _find_aggregators_in_region(self, region: str) -> List[str]:
|
153
|
+
"""Find Config aggregators in a specific region"""
|
154
|
+
try:
|
155
|
+
config_client = self.session.client('config', region_name=region)
|
156
|
+
|
157
|
+
# List configuration aggregators
|
158
|
+
response = config_client.describe_configuration_aggregators()
|
159
|
+
self.metrics['api_calls_made'] += 1
|
160
|
+
|
161
|
+
aggregators = []
|
162
|
+
for aggregator in response.get('ConfigurationAggregators', []):
|
163
|
+
aggregator_name = aggregator['ConfigurationAggregatorName']
|
164
|
+
aggregators.append(aggregator_name)
|
165
|
+
|
166
|
+
# Log aggregator details for debugging
|
167
|
+
logger.debug(f"Found aggregator {aggregator_name} in {region}")
|
168
|
+
if 'OrganizationAggregationSource' in aggregator:
|
169
|
+
org_source = aggregator['OrganizationAggregationSource']
|
170
|
+
logger.debug(f"Organization aggregator: AllAwsRegions={org_source.get('AllAwsRegions', False)}")
|
171
|
+
|
172
|
+
return aggregators
|
173
|
+
|
174
|
+
except ClientError as e:
|
175
|
+
error_code = e.response['Error']['Code']
|
176
|
+
if error_code in ['AccessDenied', 'UnauthorizedOperation']:
|
177
|
+
logger.debug(f"No Config access in {region}: {error_code}")
|
178
|
+
else:
|
179
|
+
logger.warning(f"Config API error in {region}: {e}")
|
180
|
+
return []
|
181
|
+
except Exception as e:
|
182
|
+
logger.error(f"Unexpected error in {region}: {e}")
|
183
|
+
return []
|
184
|
+
|
185
|
+
def discover_rds_snapshots_via_aggregator(self, target_account_ids: Optional[List[str]] = None) -> List[Dict]:
|
186
|
+
"""
|
187
|
+
Discover RDS snapshots using Config aggregators
|
188
|
+
|
189
|
+
Args:
|
190
|
+
target_account_ids: Specific account IDs to filter (None for all accounts)
|
191
|
+
|
192
|
+
Returns:
|
193
|
+
List of RDS snapshot dictionaries with comprehensive metadata
|
194
|
+
"""
|
195
|
+
print_header("RDS Snapshot Discovery via Config Aggregator")
|
196
|
+
|
197
|
+
all_snapshots = []
|
198
|
+
|
199
|
+
if not self.discovered_aggregators:
|
200
|
+
print_warning("No Config aggregators available - discovering now...")
|
201
|
+
self.discover_config_aggregators()
|
202
|
+
|
203
|
+
if not self.discovered_aggregators:
|
204
|
+
print_error("No Config aggregators found - cannot proceed with discovery")
|
205
|
+
return []
|
206
|
+
|
207
|
+
# Process each region's aggregators
|
208
|
+
with create_progress_bar() as progress:
|
209
|
+
total_aggregators = sum(len(aggs) for aggs in self.discovered_aggregators.values())
|
210
|
+
task_id = progress.add_task(
|
211
|
+
"Discovering RDS snapshots via Config aggregators...",
|
212
|
+
total=total_aggregators
|
213
|
+
)
|
214
|
+
|
215
|
+
for region, aggregators in self.discovered_aggregators.items():
|
216
|
+
console.print(f"\n[cyan]🔍 Processing Config aggregators in {region}[/cyan]")
|
217
|
+
|
218
|
+
for aggregator_name in aggregators:
|
219
|
+
try:
|
220
|
+
snapshots = self._query_snapshots_from_aggregator(
|
221
|
+
region, aggregator_name, target_account_ids
|
222
|
+
)
|
223
|
+
|
224
|
+
if snapshots:
|
225
|
+
all_snapshots.extend(snapshots)
|
226
|
+
unique_accounts = set(s.get('AccountId', 'unknown') for s in snapshots)
|
227
|
+
console.print(
|
228
|
+
f"[green]✓[/green] {aggregator_name}: "
|
229
|
+
f"Found {len(snapshots)} snapshots across {len(unique_accounts)} accounts"
|
230
|
+
)
|
231
|
+
else:
|
232
|
+
console.print(f"[dim]○[/dim] {aggregator_name}: No snapshots found")
|
233
|
+
|
234
|
+
except Exception as e:
|
235
|
+
print_warning(f"❌ {aggregator_name}: {str(e)}")
|
236
|
+
self.metrics['errors_encountered'] += 1
|
237
|
+
|
238
|
+
progress.advance(task_id)
|
239
|
+
|
240
|
+
# Update metrics
|
241
|
+
self.metrics['snapshots_discovered'] = len(all_snapshots)
|
242
|
+
unique_accounts = set(s.get('AccountId', 'unknown') for s in all_snapshots)
|
243
|
+
self.metrics['accounts_covered'] = len(unique_accounts)
|
244
|
+
|
245
|
+
# Summary
|
246
|
+
if all_snapshots:
|
247
|
+
print_success(
|
248
|
+
f"✅ Discovery complete: {len(all_snapshots)} RDS snapshots "
|
249
|
+
f"across {len(unique_accounts)} accounts"
|
250
|
+
)
|
251
|
+
else:
|
252
|
+
print_warning("⚠️ No RDS snapshots discovered via Config aggregators")
|
253
|
+
|
254
|
+
self.snapshot_inventory = all_snapshots
|
255
|
+
return all_snapshots
|
256
|
+
|
257
|
+
def _query_snapshots_from_aggregator(
|
258
|
+
self,
|
259
|
+
region: str,
|
260
|
+
aggregator_name: str,
|
261
|
+
target_account_ids: Optional[List[str]] = None
|
262
|
+
) -> List[Dict]:
|
263
|
+
"""Query RDS snapshots from a specific Config aggregator"""
|
264
|
+
try:
|
265
|
+
config_client = self.session.client('config', region_name=region)
|
266
|
+
|
267
|
+
# Base query for RDS DB snapshots
|
268
|
+
query_expression = """
|
269
|
+
SELECT
|
270
|
+
resourceId,
|
271
|
+
resourceName,
|
272
|
+
accountId,
|
273
|
+
awsRegion,
|
274
|
+
resourceType,
|
275
|
+
configuration,
|
276
|
+
configurationItemCaptureTime,
|
277
|
+
resourceCreationTime
|
278
|
+
WHERE
|
279
|
+
resourceType = 'AWS::RDS::DBSnapshot'
|
280
|
+
"""
|
281
|
+
|
282
|
+
# Add account filter if specified
|
283
|
+
if target_account_ids:
|
284
|
+
accounts_filter = "', '".join(target_account_ids)
|
285
|
+
query_expression += f" AND accountId IN ('{accounts_filter}')"
|
286
|
+
|
287
|
+
# Execute query with pagination support
|
288
|
+
snapshots = []
|
289
|
+
next_token = None
|
290
|
+
|
291
|
+
while True:
|
292
|
+
query_params = {
|
293
|
+
'ConfigurationAggregatorName': aggregator_name,
|
294
|
+
'Expression': query_expression,
|
295
|
+
'Limit': 100 # Maximum allowed by Config API
|
296
|
+
}
|
297
|
+
|
298
|
+
if next_token:
|
299
|
+
query_params['NextToken'] = next_token
|
300
|
+
|
301
|
+
response = config_client.select_aggregate_resource_config(**query_params)
|
302
|
+
self.metrics['api_calls_made'] += 1
|
303
|
+
|
304
|
+
# Process results
|
305
|
+
for result in response.get('Results', []):
|
306
|
+
try:
|
307
|
+
snapshot_data = json.loads(result)
|
308
|
+
processed_snapshot = self._process_config_snapshot_result(snapshot_data)
|
309
|
+
if processed_snapshot:
|
310
|
+
snapshots.append(processed_snapshot)
|
311
|
+
except json.JSONDecodeError as e:
|
312
|
+
logger.warning(f"Failed to parse Config result: {e}")
|
313
|
+
except Exception as e:
|
314
|
+
logger.warning(f"Failed to process snapshot data: {e}")
|
315
|
+
|
316
|
+
# Check for more results
|
317
|
+
next_token = response.get('NextToken')
|
318
|
+
if not next_token:
|
319
|
+
break
|
320
|
+
|
321
|
+
return snapshots
|
322
|
+
|
323
|
+
except ClientError as e:
|
324
|
+
error_code = e.response['Error']['Code']
|
325
|
+
if error_code == 'NoSuchConfigurationAggregatorException':
|
326
|
+
logger.warning(f"Aggregator {aggregator_name} not found in {region}")
|
327
|
+
elif error_code in ['AccessDenied', 'UnauthorizedOperation']:
|
328
|
+
logger.warning(f"Access denied to aggregator {aggregator_name} in {region}")
|
329
|
+
else:
|
330
|
+
logger.error(f"Config aggregator query failed: {e}")
|
331
|
+
return []
|
332
|
+
except Exception as e:
|
333
|
+
logger.error(f"Unexpected error querying aggregator {aggregator_name}: {e}")
|
334
|
+
return []
|
335
|
+
|
336
|
+
def _process_config_snapshot_result(self, config_data: Dict) -> Optional[Dict]:
|
337
|
+
"""Process Config aggregator result into standardized snapshot format"""
|
338
|
+
try:
|
339
|
+
# Extract base metadata
|
340
|
+
snapshot_info = {
|
341
|
+
'DBSnapshotIdentifier': config_data.get('resourceId', 'unknown'),
|
342
|
+
'AccountId': config_data.get('accountId', 'unknown'),
|
343
|
+
'Region': config_data.get('awsRegion', 'unknown'),
|
344
|
+
'DiscoveryMethod': 'config_aggregator',
|
345
|
+
'ConfigCaptureTime': config_data.get('configurationItemCaptureTime'),
|
346
|
+
'ResourceCreationTime': config_data.get('resourceCreationTime')
|
347
|
+
}
|
348
|
+
|
349
|
+
# Parse configuration details if available
|
350
|
+
configuration = config_data.get('configuration', {})
|
351
|
+
if isinstance(configuration, str):
|
352
|
+
try:
|
353
|
+
configuration = json.loads(configuration)
|
354
|
+
except json.JSONDecodeError:
|
355
|
+
configuration = {}
|
356
|
+
|
357
|
+
# Extract RDS-specific details
|
358
|
+
if configuration:
|
359
|
+
snapshot_info.update({
|
360
|
+
'DBInstanceIdentifier': configuration.get('dBInstanceIdentifier', 'unknown'),
|
361
|
+
'SnapshotType': configuration.get('snapshotType', 'unknown'),
|
362
|
+
'Status': configuration.get('status', 'unknown'),
|
363
|
+
'Engine': configuration.get('engine', 'unknown'),
|
364
|
+
'EngineVersion': configuration.get('engineVersion', 'unknown'),
|
365
|
+
'AllocatedStorage': configuration.get('allocatedStorage', 0),
|
366
|
+
'StorageType': configuration.get('storageType', 'unknown'),
|
367
|
+
'Encrypted': configuration.get('encrypted', False),
|
368
|
+
'SnapshotCreateTime': configuration.get('snapshotCreateTime'),
|
369
|
+
'InstanceCreateTime': configuration.get('instanceCreateTime'),
|
370
|
+
'MasterUsername': configuration.get('masterUsername', 'unknown'),
|
371
|
+
'Port': configuration.get('port', 0),
|
372
|
+
'VpcId': configuration.get('vpcId'),
|
373
|
+
'AvailabilityZone': configuration.get('availabilityZone'),
|
374
|
+
'LicenseModel': configuration.get('licenseModel', 'unknown'),
|
375
|
+
'OptionGroupName': configuration.get('optionGroupName'),
|
376
|
+
'PercentProgress': configuration.get('percentProgress', 0),
|
377
|
+
'SourceRegion': configuration.get('sourceRegion'),
|
378
|
+
'SourceDBSnapshotIdentifier': configuration.get('sourceDBSnapshotIdentifier'),
|
379
|
+
'StorageEncrypted': configuration.get('storageEncrypted', False),
|
380
|
+
'KmsKeyId': configuration.get('kmsKeyId'),
|
381
|
+
'Timezone': configuration.get('timezone'),
|
382
|
+
'IAMDatabaseAuthenticationEnabled': configuration.get('iAMDatabaseAuthenticationEnabled', False),
|
383
|
+
'ProcessorFeatures': configuration.get('processorFeatures', []),
|
384
|
+
'DbiResourceId': configuration.get('dbiResourceId'),
|
385
|
+
'TagList': configuration.get('tagList', [])
|
386
|
+
})
|
387
|
+
|
388
|
+
# Calculate age and estimated cost
|
389
|
+
if snapshot_info.get('SnapshotCreateTime'):
|
390
|
+
try:
|
391
|
+
create_time = datetime.fromisoformat(
|
392
|
+
snapshot_info['SnapshotCreateTime'].replace('Z', '+00:00')
|
393
|
+
)
|
394
|
+
age_days = (datetime.now(timezone.utc) - create_time).days
|
395
|
+
snapshot_info['AgeDays'] = age_days
|
396
|
+
|
397
|
+
# Estimate storage cost (basic calculation)
|
398
|
+
allocated_storage = snapshot_info.get('AllocatedStorage', 0)
|
399
|
+
if allocated_storage > 0:
|
400
|
+
# Basic cost estimation - $0.095 per GB-month for snapshot storage
|
401
|
+
monthly_cost = allocated_storage * 0.095
|
402
|
+
snapshot_info['EstimatedMonthlyCost'] = round(monthly_cost, 2)
|
403
|
+
snapshot_info['EstimatedAnnualCost'] = round(monthly_cost * 12, 2)
|
404
|
+
|
405
|
+
except Exception as e:
|
406
|
+
logger.debug(f"Failed to calculate snapshot age: {e}")
|
407
|
+
snapshot_info['AgeDays'] = 0
|
408
|
+
|
409
|
+
return snapshot_info
|
410
|
+
|
411
|
+
except Exception as e:
|
412
|
+
logger.warning(f"Failed to process Config snapshot result: {e}")
|
413
|
+
return None
|
414
|
+
|
415
|
+
def filter_snapshots(
|
416
|
+
self,
|
417
|
+
snapshots: List[Dict],
|
418
|
+
account_filter: Optional[List[str]] = None,
|
419
|
+
age_filter_days: Optional[int] = None,
|
420
|
+
snapshot_type_filter: Optional[str] = None,
|
421
|
+
engine_filter: Optional[str] = None
|
422
|
+
) -> List[Dict]:
|
423
|
+
"""
|
424
|
+
Apply filters to discovered snapshots
|
425
|
+
|
426
|
+
Args:
|
427
|
+
snapshots: List of snapshot dictionaries
|
428
|
+
account_filter: Filter by specific account IDs
|
429
|
+
age_filter_days: Filter snapshots older than X days
|
430
|
+
snapshot_type_filter: Filter by snapshot type (manual, automated)
|
431
|
+
engine_filter: Filter by database engine
|
432
|
+
|
433
|
+
Returns:
|
434
|
+
Filtered list of snapshots
|
435
|
+
"""
|
436
|
+
filtered_snapshots = snapshots.copy()
|
437
|
+
|
438
|
+
# Account filter
|
439
|
+
if account_filter:
|
440
|
+
filtered_snapshots = [
|
441
|
+
s for s in filtered_snapshots
|
442
|
+
if s.get('AccountId') in account_filter
|
443
|
+
]
|
444
|
+
console.print(f"[dim]Account filter: {len(filtered_snapshots)} snapshots[/dim]")
|
445
|
+
|
446
|
+
# Age filter
|
447
|
+
if age_filter_days is not None:
|
448
|
+
filtered_snapshots = [
|
449
|
+
s for s in filtered_snapshots
|
450
|
+
if s.get('AgeDays', 0) >= age_filter_days
|
451
|
+
]
|
452
|
+
console.print(f"[dim]Age filter (>{age_filter_days}d): {len(filtered_snapshots)} snapshots[/dim]")
|
453
|
+
|
454
|
+
# Snapshot type filter
|
455
|
+
if snapshot_type_filter:
|
456
|
+
filtered_snapshots = [
|
457
|
+
s for s in filtered_snapshots
|
458
|
+
if s.get('SnapshotType', '').lower() == snapshot_type_filter.lower()
|
459
|
+
]
|
460
|
+
console.print(f"[dim]Type filter ({snapshot_type_filter}): {len(filtered_snapshots)} snapshots[/dim]")
|
461
|
+
|
462
|
+
# Engine filter
|
463
|
+
if engine_filter:
|
464
|
+
filtered_snapshots = [
|
465
|
+
s for s in filtered_snapshots
|
466
|
+
if engine_filter.lower() in s.get('Engine', '').lower()
|
467
|
+
]
|
468
|
+
console.print(f"[dim]Engine filter ({engine_filter}): {len(filtered_snapshots)} snapshots[/dim]")
|
469
|
+
|
470
|
+
return filtered_snapshots
|
471
|
+
|
472
|
+
def generate_summary_report(self, snapshots: List[Dict]) -> Dict:
|
473
|
+
"""Generate comprehensive summary report of discovered snapshots"""
|
474
|
+
if not snapshots:
|
475
|
+
return {
|
476
|
+
'total_snapshots': 0,
|
477
|
+
'summary': 'No snapshots discovered'
|
478
|
+
}
|
479
|
+
|
480
|
+
# Basic statistics
|
481
|
+
total_snapshots = len(snapshots)
|
482
|
+
unique_accounts = set(s.get('AccountId', 'unknown') for s in snapshots)
|
483
|
+
unique_regions = set(s.get('Region', 'unknown') for s in snapshots)
|
484
|
+
unique_engines = set(s.get('Engine', 'unknown') for s in snapshots)
|
485
|
+
|
486
|
+
# Snapshot type breakdown
|
487
|
+
manual_snapshots = [s for s in snapshots if s.get('SnapshotType', '').lower() == 'manual']
|
488
|
+
automated_snapshots = [s for s in snapshots if s.get('SnapshotType', '').lower() == 'automated']
|
489
|
+
|
490
|
+
# Age analysis
|
491
|
+
aged_snapshots = [s for s in snapshots if s.get('AgeDays', 0) >= 90] # 3+ months
|
492
|
+
very_old_snapshots = [s for s in snapshots if s.get('AgeDays', 0) >= 180] # 6+ months
|
493
|
+
|
494
|
+
# Storage analysis
|
495
|
+
total_storage = sum(s.get('AllocatedStorage', 0) for s in snapshots)
|
496
|
+
total_estimated_cost = sum(s.get('EstimatedMonthlyCost', 0) for s in snapshots)
|
497
|
+
|
498
|
+
# Encryption analysis
|
499
|
+
encrypted_snapshots = [s for s in snapshots if s.get('Encrypted', False)]
|
500
|
+
|
501
|
+
return {
|
502
|
+
'total_snapshots': total_snapshots,
|
503
|
+
'unique_accounts': len(unique_accounts),
|
504
|
+
'unique_regions': len(unique_regions),
|
505
|
+
'unique_engines': len(unique_engines),
|
506
|
+
'account_ids': sorted(list(unique_accounts)),
|
507
|
+
'regions': sorted(list(unique_regions)),
|
508
|
+
'engines': sorted(list(unique_engines)),
|
509
|
+
'snapshot_types': {
|
510
|
+
'manual': len(manual_snapshots),
|
511
|
+
'automated': len(automated_snapshots),
|
512
|
+
'unknown': total_snapshots - len(manual_snapshots) - len(automated_snapshots)
|
513
|
+
},
|
514
|
+
'age_analysis': {
|
515
|
+
'aged_snapshots_90d': len(aged_snapshots),
|
516
|
+
'very_old_snapshots_180d': len(very_old_snapshots),
|
517
|
+
'cleanup_candidates': len([s for s in manual_snapshots if s.get('AgeDays', 0) >= 90])
|
518
|
+
},
|
519
|
+
'storage_analysis': {
|
520
|
+
'total_storage_gb': total_storage,
|
521
|
+
'estimated_monthly_cost': round(total_estimated_cost, 2),
|
522
|
+
'estimated_annual_cost': round(total_estimated_cost * 12, 2)
|
523
|
+
},
|
524
|
+
'security_analysis': {
|
525
|
+
'encrypted_snapshots': len(encrypted_snapshots),
|
526
|
+
'unencrypted_snapshots': total_snapshots - len(encrypted_snapshots),
|
527
|
+
'encryption_percentage': round((len(encrypted_snapshots) / total_snapshots) * 100, 1) if total_snapshots > 0 else 0
|
528
|
+
},
|
529
|
+
'discovery_metrics': self.metrics
|
530
|
+
}
|
531
|
+
|
532
|
+
def export_results(self, snapshots: List[Dict], output_file: str, format: str = 'csv') -> bool:
|
533
|
+
"""Export snapshot results to file"""
|
534
|
+
try:
|
535
|
+
if format.lower() == 'csv':
|
536
|
+
import csv
|
537
|
+
|
538
|
+
if not snapshots:
|
539
|
+
console.print("[yellow]No snapshots to export[/yellow]")
|
540
|
+
return False
|
541
|
+
|
542
|
+
# Get all possible keys from snapshots
|
543
|
+
all_keys = set()
|
544
|
+
for snapshot in snapshots:
|
545
|
+
all_keys.update(snapshot.keys())
|
546
|
+
|
547
|
+
fieldnames = sorted(list(all_keys))
|
548
|
+
|
549
|
+
with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
|
550
|
+
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
551
|
+
writer.writeheader()
|
552
|
+
for snapshot in snapshots:
|
553
|
+
# Convert complex objects to strings for CSV
|
554
|
+
row = {}
|
555
|
+
for key in fieldnames:
|
556
|
+
value = snapshot.get(key, '')
|
557
|
+
if isinstance(value, (list, dict)):
|
558
|
+
row[key] = json.dumps(value)
|
559
|
+
else:
|
560
|
+
row[key] = str(value) if value is not None else ''
|
561
|
+
writer.writerow(row)
|
562
|
+
|
563
|
+
elif format.lower() == 'json':
|
564
|
+
with open(output_file, 'w', encoding='utf-8') as jsonfile:
|
565
|
+
json.dump(snapshots, jsonfile, indent=2, default=str)
|
566
|
+
|
567
|
+
else:
|
568
|
+
print_error(f"Unsupported export format: {format}")
|
569
|
+
return False
|
570
|
+
|
571
|
+
print_success(f"✅ Exported {len(snapshots)} snapshots to {output_file}")
|
572
|
+
return True
|
573
|
+
|
574
|
+
except Exception as e:
|
575
|
+
print_error(f"Failed to export results: {e}")
|
576
|
+
return False
|
577
|
+
|
578
|
+
|
579
|
+
@click.command()
|
580
|
+
@click.option('--profile', help='AWS profile with Config aggregator access (overrides environment)')
|
581
|
+
@click.option('--target-accounts', multiple=True, help='Specific account IDs to analyze')
|
582
|
+
@click.option('--regions', multiple=True, help='Specific regions to check for aggregators')
|
583
|
+
@click.option('--age-filter', type=int, help='Filter snapshots older than X days')
|
584
|
+
@click.option('--snapshot-type', type=click.Choice(['manual', 'automated']), help='Filter by snapshot type')
|
585
|
+
@click.option('--engine-filter', help='Filter by database engine (partial match)')
|
586
|
+
@click.option('--output-file', default='./rds_snapshots_config_discovery.csv', help='Output file path')
|
587
|
+
@click.option('--format', type=click.Choice(['csv', 'json']), default='csv', help='Output format')
|
588
|
+
@click.option('--max-workers', type=int, default=10, help='Maximum concurrent workers')
|
589
|
+
@click.option('--summary-only', is_flag=True, help='Show only summary report')
|
590
|
+
def discover_rds_snapshots(
|
591
|
+
profile: str,
|
592
|
+
target_accounts: Tuple[str],
|
593
|
+
regions: Tuple[str],
|
594
|
+
age_filter: int,
|
595
|
+
snapshot_type: str,
|
596
|
+
engine_filter: str,
|
597
|
+
output_file: str,
|
598
|
+
format: str,
|
599
|
+
max_workers: int,
|
600
|
+
summary_only: bool
|
601
|
+
):
|
602
|
+
"""
|
603
|
+
Enhanced RDS Snapshot Discovery via AWS Config Organization Aggregator
|
604
|
+
|
605
|
+
Solves cross-account access limitations by leveraging AWS Config's organization-aggregator
|
606
|
+
to discover RDS snapshots across all enterprise accounts where direct RDS API access fails.
|
607
|
+
|
608
|
+
Examples:
|
609
|
+
# Discover all snapshots across the organization
|
610
|
+
runbooks inventory rds-snapshots-config --profile management-profile
|
611
|
+
|
612
|
+
# Target specific accounts
|
613
|
+
runbooks inventory rds-snapshots-config --target-accounts 142964829704 --target-accounts 363435891329
|
614
|
+
|
615
|
+
# Focus on old manual snapshots for cleanup
|
616
|
+
runbooks inventory rds-snapshots-config --snapshot-type manual --age-filter 90
|
617
|
+
|
618
|
+
# Export to JSON format
|
619
|
+
runbooks inventory rds-snapshots-config --format json --output-file snapshots.json
|
620
|
+
"""
|
621
|
+
try:
|
622
|
+
print_header("Enhanced RDS Snapshot Discovery via Config Aggregator", "v1.0")
|
623
|
+
|
624
|
+
# Initialize discovery engine
|
625
|
+
aggregator = RDSSnapshotConfigAggregator(
|
626
|
+
management_profile=profile,
|
627
|
+
max_workers=max_workers
|
628
|
+
)
|
629
|
+
|
630
|
+
# Override default regions if specified
|
631
|
+
if regions:
|
632
|
+
aggregator.config_regions = list(regions)
|
633
|
+
console.print(f"[dim]Using custom regions: {', '.join(regions)}[/dim]")
|
634
|
+
|
635
|
+
# Initialize session
|
636
|
+
if not aggregator.initialize_session():
|
637
|
+
return
|
638
|
+
|
639
|
+
# Discover Config aggregators
|
640
|
+
aggregator_map = aggregator.discover_config_aggregators()
|
641
|
+
if not aggregator_map:
|
642
|
+
print_error("❌ No Config aggregators found - cannot proceed with discovery")
|
643
|
+
return
|
644
|
+
|
645
|
+
# Discover RDS snapshots
|
646
|
+
target_account_list = list(target_accounts) if target_accounts else None
|
647
|
+
snapshots = aggregator.discover_rds_snapshots_via_aggregator(target_account_list)
|
648
|
+
|
649
|
+
if not snapshots:
|
650
|
+
print_warning("⚠️ No RDS snapshots discovered")
|
651
|
+
return
|
652
|
+
|
653
|
+
# Apply filters
|
654
|
+
if age_filter or snapshot_type or engine_filter:
|
655
|
+
console.print("\n[cyan]🔍 Applying filters...[/cyan]")
|
656
|
+
snapshots = aggregator.filter_snapshots(
|
657
|
+
snapshots,
|
658
|
+
account_filter=target_account_list,
|
659
|
+
age_filter_days=age_filter,
|
660
|
+
snapshot_type_filter=snapshot_type,
|
661
|
+
engine_filter=engine_filter
|
662
|
+
)
|
663
|
+
|
664
|
+
# Generate summary report
|
665
|
+
summary = aggregator.generate_summary_report(snapshots)
|
666
|
+
|
667
|
+
# Display summary
|
668
|
+
print_header("Discovery Summary")
|
669
|
+
|
670
|
+
summary_table = create_table(
|
671
|
+
title="RDS Snapshot Discovery Results",
|
672
|
+
columns=[
|
673
|
+
{"header": "Metric", "style": "cyan"},
|
674
|
+
{"header": "Value", "style": "green bold"}
|
675
|
+
]
|
676
|
+
)
|
677
|
+
|
678
|
+
summary_table.add_row("Total Snapshots", str(summary['total_snapshots']))
|
679
|
+
summary_table.add_row("Unique Accounts", str(summary['unique_accounts']))
|
680
|
+
summary_table.add_row("Unique Regions", str(summary['unique_regions']))
|
681
|
+
summary_table.add_row("Database Engines", str(summary['unique_engines']))
|
682
|
+
summary_table.add_row("Manual Snapshots", str(summary['snapshot_types']['manual']))
|
683
|
+
summary_table.add_row("Automated Snapshots", str(summary['snapshot_types']['automated']))
|
684
|
+
summary_table.add_row("Old Snapshots (90d+)", str(summary['age_analysis']['aged_snapshots_90d']))
|
685
|
+
summary_table.add_row("Cleanup Candidates", str(summary['age_analysis']['cleanup_candidates']))
|
686
|
+
summary_table.add_row("Total Storage (GB)", str(summary['storage_analysis']['total_storage_gb']))
|
687
|
+
summary_table.add_row("Est. Monthly Cost", format_cost(summary['storage_analysis']['estimated_monthly_cost']))
|
688
|
+
summary_table.add_row("Est. Annual Cost", format_cost(summary['storage_analysis']['estimated_annual_cost']))
|
689
|
+
summary_table.add_row("Encrypted Snapshots", f"{summary['security_analysis']['encrypted_snapshots']} ({summary['security_analysis']['encryption_percentage']}%)")
|
690
|
+
|
691
|
+
console.print(summary_table)
|
692
|
+
|
693
|
+
# Discovery metrics
|
694
|
+
metrics_table = create_table(
|
695
|
+
title="Discovery Performance Metrics",
|
696
|
+
columns=[
|
697
|
+
{"header": "Metric", "style": "blue"},
|
698
|
+
{"header": "Count", "style": "yellow"}
|
699
|
+
]
|
700
|
+
)
|
701
|
+
|
702
|
+
metrics = summary['discovery_metrics']
|
703
|
+
metrics_table.add_row("Config Aggregators Found", str(metrics['aggregators_found']))
|
704
|
+
metrics_table.add_row("Regions Scanned", str(metrics['regions_scanned']))
|
705
|
+
metrics_table.add_row("API Calls Made", str(metrics['api_calls_made']))
|
706
|
+
metrics_table.add_row("Errors Encountered", str(metrics['errors_encountered']))
|
707
|
+
|
708
|
+
console.print(metrics_table)
|
709
|
+
|
710
|
+
# Account details
|
711
|
+
if summary['account_ids'] and len(summary['account_ids']) <= 20: # Don't flood output
|
712
|
+
console.print(f"\n[cyan]📋 Accounts with RDS snapshots:[/cyan]")
|
713
|
+
for account_id in summary['account_ids']:
|
714
|
+
account_snapshots = [s for s in snapshots if s.get('AccountId') == account_id]
|
715
|
+
console.print(f" [green]•[/green] {account_id}: {len(account_snapshots)} snapshots")
|
716
|
+
|
717
|
+
# Export results unless summary-only
|
718
|
+
if not summary_only:
|
719
|
+
if aggregator.export_results(snapshots, output_file, format):
|
720
|
+
console.print(f"\n[green]📁 Results exported to: {output_file}[/green]")
|
721
|
+
|
722
|
+
# Configuration recommendations
|
723
|
+
console.print(f"\n[cyan]💡 Configuration Details:[/cyan]")
|
724
|
+
for region, aggregators in aggregator_map.items():
|
725
|
+
console.print(f" [blue]•[/blue] {region}: {', '.join(aggregators)}")
|
726
|
+
|
727
|
+
if summary['age_analysis']['cleanup_candidates'] > 0:
|
728
|
+
print_warning(
|
729
|
+
f"🎯 Found {summary['age_analysis']['cleanup_candidates']} manual snapshots "
|
730
|
+
f"older than 90 days - consider cleanup for cost optimization"
|
731
|
+
)
|
732
|
+
|
733
|
+
if summary['security_analysis']['unencrypted_snapshots'] > 0:
|
734
|
+
print_warning(
|
735
|
+
f"🔒 Found {summary['security_analysis']['unencrypted_snapshots']} unencrypted snapshots "
|
736
|
+
f"- review for security compliance"
|
737
|
+
)
|
738
|
+
|
739
|
+
except Exception as e:
|
740
|
+
print_error(f"Discovery failed: {e}")
|
741
|
+
raise click.ClickException(str(e))
|
742
|
+
|
743
|
+
|
744
|
+
if __name__ == "__main__":
|
745
|
+
discover_rds_snapshots()
|