runbooks 0.7.6__py3-none-any.whl → 0.7.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/base.py +5 -1
- runbooks/cfat/assessment/compliance.py +847 -0
- runbooks/finops/cli.py +62 -0
- runbooks/finops/dashboard_runner.py +632 -161
- runbooks/finops/helpers.py +492 -61
- runbooks/finops/optimizer.py +822 -0
- runbooks/inventory/collectors/aws_comprehensive.py +435 -0
- runbooks/inventory/discovery.md +1 -1
- runbooks/main.py +158 -12
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/METADATA +1 -1
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/RECORD +15 -12
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/WHEEL +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/entry_points.txt +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.7.6.dist-info → runbooks-0.7.7.dist-info}/top_level.txt +0 -0
@@ -1,17 +1,21 @@
|
|
1
1
|
import argparse
|
2
|
+
import os
|
2
3
|
from collections import defaultdict
|
3
4
|
from typing import Any, Dict, List, Optional, Tuple
|
4
5
|
|
5
6
|
import boto3
|
6
7
|
from rich import box
|
7
8
|
from rich.console import Console
|
9
|
+
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn
|
8
10
|
from rich.progress import track
|
9
11
|
from rich.status import Status
|
10
12
|
from rich.table import Column, Table
|
11
13
|
|
12
14
|
from runbooks.finops.aws_client import (
|
15
|
+
ec2_summary,
|
13
16
|
get_accessible_regions,
|
14
17
|
get_account_id,
|
18
|
+
get_aws_profiles,
|
15
19
|
get_budgets,
|
16
20
|
get_stopped_instances,
|
17
21
|
get_untagged_resources,
|
@@ -23,6 +27,10 @@ from runbooks.finops.cost_processor import (
|
|
23
27
|
export_to_json,
|
24
28
|
get_cost_data,
|
25
29
|
get_trend,
|
30
|
+
change_in_total_cost,
|
31
|
+
format_budget_info,
|
32
|
+
format_ec2_summary,
|
33
|
+
process_service_costs,
|
26
34
|
)
|
27
35
|
from runbooks.finops.helpers import (
|
28
36
|
clean_rich_tags,
|
@@ -31,6 +39,7 @@ from runbooks.finops.helpers import (
|
|
31
39
|
export_audit_report_to_pdf,
|
32
40
|
export_cost_dashboard_to_pdf,
|
33
41
|
export_trend_data_to_json,
|
42
|
+
generate_pdca_improvement_report,
|
34
43
|
)
|
35
44
|
from runbooks.finops.profile_processor import (
|
36
45
|
process_combined_profiles,
|
@@ -42,6 +51,153 @@ from runbooks.finops.visualisations import create_trend_bars
|
|
42
51
|
console = Console()
|
43
52
|
|
44
53
|
|
54
|
+
def _get_profile_for_operation(operation_type: str, default_profile: str) -> str:
|
55
|
+
"""
|
56
|
+
Get the appropriate AWS profile based on operation type.
|
57
|
+
|
58
|
+
Args:
|
59
|
+
operation_type: Type of operation ('billing', 'management', 'operational')
|
60
|
+
default_profile: Default profile to fall back to
|
61
|
+
|
62
|
+
Returns:
|
63
|
+
str: Profile name to use for the operation
|
64
|
+
"""
|
65
|
+
profile_map = {
|
66
|
+
'billing': os.getenv('BILLING_PROFILE'),
|
67
|
+
'management': os.getenv('MANAGEMENT_PROFILE'),
|
68
|
+
'operational': os.getenv('CENTRALISED_OPS_PROFILE')
|
69
|
+
}
|
70
|
+
|
71
|
+
profile = profile_map.get(operation_type)
|
72
|
+
if profile:
|
73
|
+
# Verify profile exists
|
74
|
+
available_profiles = boto3.Session().available_profiles
|
75
|
+
if profile in available_profiles:
|
76
|
+
console.log(f"[dim cyan]Using {operation_type} profile: {profile}[/]")
|
77
|
+
return profile
|
78
|
+
else:
|
79
|
+
console.log(f"[yellow]Warning: {operation_type.title()} profile '{profile}' not found in AWS config. Using default: {default_profile}[/]")
|
80
|
+
|
81
|
+
return default_profile
|
82
|
+
|
83
|
+
|
84
|
+
def _create_cost_session(profile: str) -> boto3.Session:
|
85
|
+
"""
|
86
|
+
Create a boto3 session specifically for cost operations.
|
87
|
+
Uses BILLING_PROFILE if available, falls back to provided profile.
|
88
|
+
|
89
|
+
Args:
|
90
|
+
profile: Default profile to use
|
91
|
+
|
92
|
+
Returns:
|
93
|
+
boto3.Session: Session configured for cost operations
|
94
|
+
"""
|
95
|
+
cost_profile = _get_profile_for_operation('billing', profile)
|
96
|
+
return boto3.Session(profile_name=cost_profile)
|
97
|
+
|
98
|
+
|
99
|
+
def _create_management_session(profile: str) -> boto3.Session:
|
100
|
+
"""
|
101
|
+
Create a boto3 session specifically for management operations.
|
102
|
+
Uses MANAGEMENT_PROFILE if available, falls back to provided profile.
|
103
|
+
|
104
|
+
Args:
|
105
|
+
profile: Default profile to use
|
106
|
+
|
107
|
+
Returns:
|
108
|
+
boto3.Session: Session configured for management operations
|
109
|
+
"""
|
110
|
+
mgmt_profile = _get_profile_for_operation('management', profile)
|
111
|
+
return boto3.Session(profile_name=mgmt_profile)
|
112
|
+
|
113
|
+
|
114
|
+
def _create_operational_session(profile: str) -> boto3.Session:
|
115
|
+
"""
|
116
|
+
Create a boto3 session specifically for operational tasks.
|
117
|
+
Uses CENTRALISED_OPS_PROFILE if available, falls back to provided profile.
|
118
|
+
|
119
|
+
Args:
|
120
|
+
profile: Default profile to use
|
121
|
+
|
122
|
+
Returns:
|
123
|
+
boto3.Session: Session configured for operational tasks
|
124
|
+
"""
|
125
|
+
ops_profile = _get_profile_for_operation('operational', profile)
|
126
|
+
return boto3.Session(profile_name=ops_profile)
|
127
|
+
|
128
|
+
|
129
|
+
def _calculate_risk_score(untagged, stopped, unused_vols, unused_eips, budget_data):
|
130
|
+
"""Calculate risk score based on audit findings for PDCA tracking."""
|
131
|
+
score = 0
|
132
|
+
|
133
|
+
# Untagged resources (high risk for compliance)
|
134
|
+
untagged_count = sum(len(ids) for region_map in untagged.values() for ids in region_map.values())
|
135
|
+
score += untagged_count * 2 # High weight for untagged
|
136
|
+
|
137
|
+
# Stopped instances (medium risk for cost)
|
138
|
+
stopped_count = sum(len(ids) for ids in stopped.values())
|
139
|
+
score += stopped_count * 1
|
140
|
+
|
141
|
+
# Unused volumes (medium risk for cost)
|
142
|
+
volume_count = sum(len(ids) for ids in unused_vols.values())
|
143
|
+
score += volume_count * 1
|
144
|
+
|
145
|
+
# Unused EIPs (high risk for cost)
|
146
|
+
eip_count = sum(len(ids) for ids in unused_eips.values())
|
147
|
+
score += eip_count * 3 # High cost impact
|
148
|
+
|
149
|
+
# Budget overruns (critical risk)
|
150
|
+
overruns = len([b for b in budget_data if b["actual"] > b["limit"]])
|
151
|
+
score += overruns * 5 # Critical weight
|
152
|
+
|
153
|
+
return score
|
154
|
+
|
155
|
+
|
156
|
+
def _format_risk_score(score):
|
157
|
+
"""Format risk score with visual indicators."""
|
158
|
+
if score == 0:
|
159
|
+
return "[bright_green]🟢 LOW\n(0)[/]"
|
160
|
+
elif score <= 10:
|
161
|
+
return f"[yellow]🟡 MEDIUM\n({score})[/]"
|
162
|
+
elif score <= 25:
|
163
|
+
return f"[orange1]🟠 HIGH\n({score})[/]"
|
164
|
+
else:
|
165
|
+
return f"[bright_red]🔴 CRITICAL\n({score})[/]"
|
166
|
+
|
167
|
+
|
168
|
+
def _display_pdca_summary(pdca_metrics):
|
169
|
+
"""Display PDCA improvement summary with actionable insights."""
|
170
|
+
if not pdca_metrics:
|
171
|
+
return
|
172
|
+
|
173
|
+
total_risk = sum(m["risk_score"] for m in pdca_metrics)
|
174
|
+
avg_risk = total_risk / len(pdca_metrics)
|
175
|
+
|
176
|
+
high_risk_accounts = [m for m in pdca_metrics if m["risk_score"] > 25]
|
177
|
+
total_untagged = sum(m["untagged_count"] for m in pdca_metrics)
|
178
|
+
total_unused_eips = sum(m["unused_eips_count"] for m in pdca_metrics)
|
179
|
+
|
180
|
+
summary_table = Table(
|
181
|
+
title="🎯 PDCA Continuous Improvement Metrics",
|
182
|
+
box=box.SIMPLE,
|
183
|
+
style="cyan"
|
184
|
+
)
|
185
|
+
summary_table.add_column("Metric", style="bold")
|
186
|
+
summary_table.add_column("Value", justify="right")
|
187
|
+
summary_table.add_column("Action Required", style="yellow")
|
188
|
+
|
189
|
+
summary_table.add_row("Average Risk Score", f"{avg_risk:.1f}",
|
190
|
+
"✅ Good" if avg_risk < 10 else "⚠️ Review Required")
|
191
|
+
summary_table.add_row("High-Risk Accounts", str(len(high_risk_accounts)),
|
192
|
+
"🔴 Immediate Action" if high_risk_accounts else "✅ Good")
|
193
|
+
summary_table.add_row("Total Untagged Resources", str(total_untagged),
|
194
|
+
"📋 Tag Management" if total_untagged > 50 else "✅ Good")
|
195
|
+
summary_table.add_row("Total Unused EIPs", str(total_unused_eips),
|
196
|
+
"💰 Cost Optimization" if total_unused_eips > 5 else "✅ Good")
|
197
|
+
|
198
|
+
console.print(summary_table)
|
199
|
+
|
200
|
+
|
45
201
|
def _initialize_profiles(
|
46
202
|
args: argparse.Namespace,
|
47
203
|
) -> Tuple[List[str], Optional[List[str]], Optional[int]]:
|
@@ -74,102 +230,173 @@ def _initialize_profiles(
|
|
74
230
|
|
75
231
|
|
76
232
|
def _run_audit_report(profiles_to_use: List[str], args: argparse.Namespace) -> None:
|
77
|
-
"""Generate and export an audit report."""
|
78
|
-
console.print("[bold bright_cyan]Preparing
|
233
|
+
"""Generate and export an audit report with PDCA continuous improvement."""
|
234
|
+
console.print("[bold bright_cyan]🔍 PLAN: Preparing comprehensive audit report...[/]")
|
235
|
+
|
236
|
+
# Display multi-profile configuration
|
237
|
+
billing_profile = os.getenv('BILLING_PROFILE')
|
238
|
+
mgmt_profile = os.getenv('MANAGEMENT_PROFILE')
|
239
|
+
ops_profile = os.getenv('CENTRALISED_OPS_PROFILE')
|
240
|
+
|
241
|
+
if any([billing_profile, mgmt_profile, ops_profile]):
|
242
|
+
console.print("[dim cyan]Multi-profile configuration detected:[/]")
|
243
|
+
if billing_profile:
|
244
|
+
console.print(f"[dim cyan] • Billing operations: {billing_profile}[/]")
|
245
|
+
if mgmt_profile:
|
246
|
+
console.print(f"[dim cyan] • Management operations: {mgmt_profile}[/]")
|
247
|
+
if ops_profile:
|
248
|
+
console.print(f"[dim cyan] • Operational tasks: {ops_profile}[/]")
|
249
|
+
console.print()
|
250
|
+
|
251
|
+
# Enhanced table with better visual hierarchy
|
79
252
|
table = Table(
|
80
|
-
Column("Profile", justify="center"),
|
81
|
-
Column("Account ID", justify="center"),
|
82
|
-
Column("Untagged Resources"),
|
83
|
-
Column("Stopped EC2 Instances"),
|
84
|
-
Column("Unused Volumes"),
|
85
|
-
Column("Unused EIPs"),
|
86
|
-
Column("Budget Alerts"),
|
87
|
-
|
253
|
+
Column("Profile", justify="center", style="bold magenta"),
|
254
|
+
Column("Account ID", justify="center", style="dim"),
|
255
|
+
Column("Untagged Resources", style="yellow"),
|
256
|
+
Column("Stopped EC2 Instances", style="red"),
|
257
|
+
Column("Unused Volumes", style="orange1"),
|
258
|
+
Column("Unused EIPs", style="cyan"),
|
259
|
+
Column("Budget Alerts", style="bright_red"),
|
260
|
+
Column("Risk Score", justify="center", style="bold"),
|
261
|
+
title="🎯 AWS FinOps Audit Report - PDCA Enhanced",
|
88
262
|
show_lines=True,
|
89
|
-
box=box.
|
263
|
+
box=box.ROUNDED,
|
90
264
|
style="bright_cyan",
|
265
|
+
caption="🚀 PDCA Cycle: Plan → Do → Check → Act",
|
91
266
|
)
|
92
267
|
|
93
268
|
audit_data = []
|
94
269
|
raw_audit_data = []
|
270
|
+
pdca_metrics = [] # New: Track PDCA improvement metrics
|
95
271
|
nl = "\n"
|
96
272
|
comma_nl = ",\n"
|
97
273
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
274
|
+
console.print("[bold green]⚙️ DO: Collecting audit data across profiles...[/]")
|
275
|
+
|
276
|
+
# Create progress tracker for enhanced user experience (v2.2.3 reference)
|
277
|
+
with Progress(
|
278
|
+
SpinnerColumn(),
|
279
|
+
TextColumn("[progress.description]{task.description}"),
|
280
|
+
BarColumn(),
|
281
|
+
TaskProgressColumn(),
|
282
|
+
TimeElapsedColumn(),
|
283
|
+
console=console,
|
284
|
+
transient=True
|
285
|
+
) as progress:
|
286
|
+
task = progress.add_task("Collecting audit data", total=len(profiles_to_use))
|
287
|
+
|
288
|
+
for profile in profiles_to_use:
|
289
|
+
progress.update(task, description=f"Processing profile: {profile}")
|
290
|
+
|
291
|
+
# Use operational session for resource discovery
|
292
|
+
ops_session = _create_operational_session(profile)
|
293
|
+
# Use management session for account and governance operations
|
294
|
+
mgmt_session = _create_management_session(profile)
|
295
|
+
# Use billing session for cost and budget operations
|
296
|
+
billing_session = _create_cost_session(profile)
|
297
|
+
|
298
|
+
account_id = get_account_id(mgmt_session) or "Unknown"
|
299
|
+
regions = args.regions or get_accessible_regions(ops_session)
|
102
300
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
301
|
+
try:
|
302
|
+
# Use operational session for resource discovery
|
303
|
+
untagged = get_untagged_resources(ops_session, regions)
|
304
|
+
anomalies = []
|
305
|
+
for service, region_map in untagged.items():
|
306
|
+
if region_map:
|
307
|
+
service_block = f"[bright_yellow]{service}[/]:\n"
|
308
|
+
for region, ids in region_map.items():
|
309
|
+
if ids:
|
310
|
+
ids_block = "\n".join(f"[orange1]{res_id}[/]" for res_id in ids)
|
311
|
+
service_block += f"\n{region}:\n{ids_block}\n"
|
312
|
+
anomalies.append(service_block)
|
313
|
+
if not any(region_map for region_map in untagged.values()):
|
314
|
+
anomalies = ["None"]
|
315
|
+
except Exception as e:
|
316
|
+
anomalies = [f"Error: {str(e)}"]
|
317
|
+
|
318
|
+
# Use operational session for EC2 and resource operations
|
319
|
+
stopped = get_stopped_instances(ops_session, regions)
|
320
|
+
stopped_list = [f"{r}:\n[gold1]{nl.join(ids)}[/]" for r, ids in stopped.items()] or ["None"]
|
321
|
+
|
322
|
+
unused_vols = get_unused_volumes(ops_session, regions)
|
323
|
+
vols_list = [f"{r}:\n[dark_orange]{nl.join(ids)}[/]" for r, ids in unused_vols.items()] or ["None"]
|
324
|
+
|
325
|
+
unused_eips = get_unused_eips(ops_session, regions)
|
326
|
+
eips_list = [f"{r}:\n{comma_nl.join(ids)}" for r, ids in unused_eips.items()] or ["None"]
|
327
|
+
|
328
|
+
# Use billing session for budget data
|
329
|
+
budget_data = get_budgets(billing_session)
|
330
|
+
alerts = []
|
331
|
+
for b in budget_data:
|
332
|
+
if b["actual"] > b["limit"]:
|
333
|
+
alerts.append(f"[red1]{b['name']}[/]: ${b['actual']:.2f} > ${b['limit']:.2f}")
|
334
|
+
if not alerts:
|
335
|
+
alerts = ["✅ No budgets exceeded"]
|
336
|
+
|
337
|
+
# Calculate risk score for PDCA improvement tracking
|
338
|
+
risk_score = _calculate_risk_score(untagged, stopped, unused_vols, unused_eips, budget_data)
|
339
|
+
risk_display = _format_risk_score(risk_score)
|
340
|
+
|
341
|
+
# Track PDCA metrics
|
342
|
+
pdca_metrics.append({
|
138
343
|
"profile": profile,
|
139
344
|
"account_id": account_id,
|
140
|
-
"
|
141
|
-
"
|
142
|
-
"
|
143
|
-
"
|
144
|
-
"
|
145
|
-
|
146
|
-
|
345
|
+
"risk_score": risk_score,
|
346
|
+
"untagged_count": sum(len(ids) for region_map in untagged.values() for ids in region_map.values()),
|
347
|
+
"stopped_count": sum(len(ids) for ids in stopped.values()),
|
348
|
+
"unused_volumes_count": sum(len(ids) for ids in unused_vols.values()),
|
349
|
+
"unused_eips_count": sum(len(ids) for ids in unused_eips.values()),
|
350
|
+
"budget_overruns": len([b for b in budget_data if b["actual"] > b["limit"]])
|
351
|
+
})
|
352
|
+
|
353
|
+
audit_data.append(
|
354
|
+
{
|
355
|
+
"profile": profile,
|
356
|
+
"account_id": account_id,
|
357
|
+
"untagged_resources": clean_rich_tags("\n".join(anomalies)),
|
358
|
+
"stopped_instances": clean_rich_tags("\n".join(stopped_list)),
|
359
|
+
"unused_volumes": clean_rich_tags("\n".join(vols_list)),
|
360
|
+
"unused_eips": clean_rich_tags("\n".join(eips_list)),
|
361
|
+
"budget_alerts": clean_rich_tags("\n".join(alerts)),
|
362
|
+
"risk_score": risk_score,
|
363
|
+
}
|
364
|
+
)
|
147
365
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
366
|
+
# Data for JSON which includes raw audit data
|
367
|
+
raw_audit_data.append(
|
368
|
+
{
|
369
|
+
"profile": profile,
|
370
|
+
"account_id": account_id,
|
371
|
+
"untagged_resources": untagged,
|
372
|
+
"stopped_instances": stopped,
|
373
|
+
"unused_volumes": unused_vols,
|
374
|
+
"unused_eips": unused_eips,
|
375
|
+
"budget_alerts": budget_data,
|
376
|
+
}
|
377
|
+
)
|
160
378
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
379
|
+
table.add_row(
|
380
|
+
f"[dark_magenta]{profile}[/]",
|
381
|
+
account_id,
|
382
|
+
"\n".join(anomalies),
|
383
|
+
"\n".join(stopped_list),
|
384
|
+
"\n".join(vols_list),
|
385
|
+
"\n".join(eips_list),
|
386
|
+
"\n".join(alerts),
|
387
|
+
risk_display,
|
388
|
+
)
|
389
|
+
|
390
|
+
progress.advance(task)
|
170
391
|
console.print(table)
|
171
|
-
|
172
|
-
|
392
|
+
|
393
|
+
# CHECK phase: Display PDCA improvement metrics
|
394
|
+
console.print("\n[bold yellow]📊 CHECK: PDCA Improvement Analysis[/]")
|
395
|
+
_display_pdca_summary(pdca_metrics)
|
396
|
+
|
397
|
+
console.print("[bold bright_cyan]📝 Note: Dashboard scans EC2, RDS, Lambda, ELBv2 resources across all accessible regions.\n[/]")
|
398
|
+
|
399
|
+
# ACT phase: Export reports with PDCA enhancements
|
173
400
|
if args.report_name: # Ensure report_name is provided for any export
|
174
401
|
if args.report_type:
|
175
402
|
for report_type in args.report_type:
|
@@ -184,59 +411,97 @@ def _run_audit_report(profiles_to_use: List[str], args: argparse.Namespace) -> N
|
|
184
411
|
elif report_type == "pdf":
|
185
412
|
pdf_path = export_audit_report_to_pdf(audit_data, args.report_name, args.dir)
|
186
413
|
if pdf_path:
|
187
|
-
console.print(f"[bright_green]Successfully exported to PDF format: {pdf_path}[/]")
|
414
|
+
console.print(f"[bright_green]✅ Successfully exported to PDF format: {pdf_path}[/]")
|
415
|
+
|
416
|
+
# Generate PDCA improvement report
|
417
|
+
console.print("\n[bold cyan]🎯 ACT: Generating PDCA improvement recommendations...[/]")
|
418
|
+
pdca_path = generate_pdca_improvement_report(pdca_metrics, args.report_name, args.dir)
|
419
|
+
if pdca_path:
|
420
|
+
console.print(f"[bright_green]🚀 PDCA improvement report saved: {pdca_path}[/]")
|
188
421
|
|
189
422
|
|
190
423
|
def _run_trend_analysis(profiles_to_use: List[str], args: argparse.Namespace) -> None:
|
191
|
-
"""Analyze and display cost trends."""
|
424
|
+
"""Analyze and display cost trends with multi-profile support."""
|
192
425
|
console.print("[bold bright_cyan]Analysing cost trends...[/]")
|
426
|
+
|
427
|
+
# Display billing profile information
|
428
|
+
billing_profile = os.getenv('BILLING_PROFILE')
|
429
|
+
if billing_profile:
|
430
|
+
console.print(f"[dim cyan]Using billing profile for cost data: {billing_profile}[/]")
|
431
|
+
|
193
432
|
raw_trend_data = []
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
433
|
+
|
434
|
+
# Enhanced progress tracking for trend analysis
|
435
|
+
with Progress(
|
436
|
+
SpinnerColumn(),
|
437
|
+
TextColumn("[progress.description]{task.description}"),
|
438
|
+
BarColumn(),
|
439
|
+
TaskProgressColumn(),
|
440
|
+
TimeElapsedColumn(),
|
441
|
+
console=console,
|
442
|
+
transient=True
|
443
|
+
) as progress:
|
444
|
+
if args.combine:
|
445
|
+
account_profiles = defaultdict(list)
|
446
|
+
task1 = progress.add_task("Grouping profiles by account", total=len(profiles_to_use))
|
447
|
+
|
448
|
+
for profile in profiles_to_use:
|
449
|
+
try:
|
450
|
+
# Use management session to get account ID
|
451
|
+
session = _create_management_session(profile)
|
452
|
+
account_id = get_account_id(session)
|
453
|
+
if account_id:
|
454
|
+
account_profiles[account_id].append(profile)
|
455
|
+
except Exception as e:
|
456
|
+
console.print(f"[red]Error checking account ID for profile {profile}: {str(e)}[/]")
|
457
|
+
progress.advance(task1)
|
458
|
+
|
459
|
+
task2 = progress.add_task("Fetching cost trends", total=len(account_profiles))
|
460
|
+
for account_id, profiles in account_profiles.items():
|
461
|
+
progress.update(task2, description=f"Fetching trends for account: {account_id}")
|
462
|
+
try:
|
463
|
+
primary_profile = profiles[0]
|
464
|
+
# Use billing session for cost trend data
|
465
|
+
cost_session = _create_cost_session(primary_profile)
|
466
|
+
cost_data = get_trend(cost_session, args.tag)
|
467
|
+
trend_data = cost_data.get("monthly_costs")
|
468
|
+
|
469
|
+
if not trend_data:
|
470
|
+
console.print(f"[yellow]No trend data available for account {account_id}[/]")
|
471
|
+
continue
|
472
|
+
|
473
|
+
profile_list = ", ".join(profiles)
|
474
|
+
console.print(f"\n[bright_yellow]Account: {account_id} (Profiles: {profile_list})[/]")
|
475
|
+
raw_trend_data.append(cost_data)
|
476
|
+
create_trend_bars(trend_data)
|
477
|
+
except Exception as e:
|
478
|
+
console.print(f"[red]Error getting trend for account {account_id}: {str(e)}[/]")
|
479
|
+
progress.advance(task2)
|
204
480
|
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
account_id = cost_data.get("account_id", "Unknown")
|
230
|
-
|
231
|
-
if not trend_data:
|
232
|
-
console.print(f"[yellow]No trend data available for profile {profile}[/]")
|
233
|
-
continue
|
234
|
-
|
235
|
-
console.print(f"\n[bright_yellow]Account: {account_id} (Profile: {profile})[/]")
|
236
|
-
raw_trend_data.append(cost_data)
|
237
|
-
create_trend_bars(trend_data)
|
238
|
-
except Exception as e:
|
239
|
-
console.print(f"[red]Error getting trend for profile {profile}: {str(e)}[/]")
|
481
|
+
else:
|
482
|
+
task3 = progress.add_task("Fetching individual trends", total=len(profiles_to_use))
|
483
|
+
for profile in profiles_to_use:
|
484
|
+
progress.update(task3, description=f"Processing profile: {profile}")
|
485
|
+
try:
|
486
|
+
# Use billing session for cost data
|
487
|
+
cost_session = _create_cost_session(profile)
|
488
|
+
# Use management session for account ID
|
489
|
+
mgmt_session = _create_management_session(profile)
|
490
|
+
|
491
|
+
cost_data = get_trend(cost_session, args.tag)
|
492
|
+
trend_data = cost_data.get("monthly_costs")
|
493
|
+
account_id = get_account_id(mgmt_session) or cost_data.get("account_id", "Unknown")
|
494
|
+
|
495
|
+
if not trend_data:
|
496
|
+
console.print(f"[yellow]No trend data available for profile {profile}[/]")
|
497
|
+
continue
|
498
|
+
|
499
|
+
console.print(f"\n[bright_yellow]Account: {account_id} (Profile: {profile})[/]")
|
500
|
+
raw_trend_data.append(cost_data)
|
501
|
+
create_trend_bars(trend_data)
|
502
|
+
except Exception as e:
|
503
|
+
console.print(f"[red]Error getting trend for profile {profile}: {str(e)}[/]")
|
504
|
+
progress.advance(task3)
|
240
505
|
|
241
506
|
if raw_trend_data and args.report_name and args.report_type:
|
242
507
|
if "json" in args.report_type:
|
@@ -246,10 +511,11 @@ def _run_trend_analysis(profiles_to_use: List[str], args: argparse.Namespace) ->
|
|
246
511
|
|
247
512
|
|
248
513
|
def _get_display_table_period_info(profiles_to_use: List[str], time_range: Optional[int]) -> Tuple[str, str, str, str]:
|
249
|
-
"""Get period information for the display table."""
|
514
|
+
"""Get period information for the display table using appropriate billing profile."""
|
250
515
|
if profiles_to_use:
|
251
516
|
try:
|
252
|
-
|
517
|
+
# Use billing session for cost data period information
|
518
|
+
sample_session = _create_cost_session(profiles_to_use[0])
|
253
519
|
sample_cost_data = get_cost_data(sample_session, time_range)
|
254
520
|
previous_period_name = sample_cost_data.get("previous_period_name", "Last Month Due")
|
255
521
|
current_period_name = sample_cost_data.get("current_period_name", "Current Month Cost")
|
@@ -342,42 +608,219 @@ def _generate_dashboard_data(
|
|
342
608
|
args: argparse.Namespace,
|
343
609
|
table: Table,
|
344
610
|
) -> List[ProfileData]:
|
345
|
-
"""Fetch, process, and prepare the main dashboard data."""
|
611
|
+
"""Fetch, process, and prepare the main dashboard data with multi-profile support."""
|
346
612
|
export_data: List[ProfileData] = []
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
613
|
+
|
614
|
+
# Enhanced progress tracking with v2.2.3 style progress bar
|
615
|
+
with Progress(
|
616
|
+
SpinnerColumn(),
|
617
|
+
TextColumn("[progress.description]{task.description}"),
|
618
|
+
BarColumn(complete_style="bright_green", finished_style="bright_green"),
|
619
|
+
TaskProgressColumn(),
|
620
|
+
TimeElapsedColumn(),
|
621
|
+
console=console,
|
622
|
+
transient=False # Keep progress visible
|
623
|
+
) as progress:
|
624
|
+
|
625
|
+
if args.combine:
|
626
|
+
account_profiles = defaultdict(list)
|
627
|
+
grouping_task = progress.add_task("Grouping profiles by account", total=len(profiles_to_use))
|
628
|
+
|
629
|
+
for profile in profiles_to_use:
|
630
|
+
progress.update(grouping_task, description=f"Checking account for profile: {profile}")
|
631
|
+
try:
|
632
|
+
# Use management session for account identification
|
633
|
+
mgmt_session = _create_management_session(profile)
|
634
|
+
current_account_id = get_account_id(mgmt_session)
|
635
|
+
if current_account_id:
|
636
|
+
account_profiles[current_account_id].append(profile)
|
637
|
+
else:
|
638
|
+
console.log(f"[yellow]Could not determine account ID for profile {profile}[/]")
|
639
|
+
except Exception as e:
|
640
|
+
console.log(f"[bold red]Error checking account ID for profile {profile}: {str(e)}[/]")
|
641
|
+
progress.advance(grouping_task)
|
642
|
+
|
643
|
+
# Process combined profiles with enhanced progress tracking
|
644
|
+
processing_task = progress.add_task("Processing account data", total=len(account_profiles))
|
645
|
+
for account_id_key, profiles_list in account_profiles.items():
|
646
|
+
progress.update(processing_task, description=f"Processing account: {account_id_key}")
|
647
|
+
|
648
|
+
if len(profiles_list) > 1:
|
649
|
+
profile_data = _process_combined_profiles_enhanced(
|
650
|
+
account_id_key, profiles_list, user_regions, time_range, args.tag
|
651
|
+
)
|
652
|
+
else:
|
653
|
+
profile_data = _process_single_profile_enhanced(profiles_list[0], user_regions, time_range, args.tag)
|
654
|
+
export_data.append(profile_data)
|
655
|
+
add_profile_to_table(table, profile_data)
|
656
|
+
progress.advance(processing_task)
|
657
|
+
|
369
658
|
else:
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
659
|
+
# Process individual profiles with enhanced progress tracking
|
660
|
+
individual_task = progress.add_task("Processing individual profiles", total=len(profiles_to_use))
|
661
|
+
for profile in profiles_to_use:
|
662
|
+
progress.update(individual_task, description=f"Processing profile: {profile}")
|
663
|
+
profile_data = _process_single_profile_enhanced(profile, user_regions, time_range, args.tag)
|
664
|
+
export_data.append(profile_data)
|
665
|
+
add_profile_to_table(table, profile_data)
|
666
|
+
progress.advance(individual_task)
|
667
|
+
|
378
668
|
return export_data
|
379
669
|
|
380
670
|
|
671
|
+
def _process_single_profile_enhanced(
|
672
|
+
profile: str,
|
673
|
+
user_regions: Optional[List[str]] = None,
|
674
|
+
time_range: Optional[int] = None,
|
675
|
+
tag: Optional[List[str]] = None,
|
676
|
+
) -> ProfileData:
|
677
|
+
"""
|
678
|
+
Enhanced single profile processing with multi-profile session support.
|
679
|
+
Uses appropriate sessions for different operations: billing, management, operational.
|
680
|
+
"""
|
681
|
+
try:
|
682
|
+
# Use billing session for cost data
|
683
|
+
cost_session = _create_cost_session(profile)
|
684
|
+
cost_data = get_cost_data(cost_session, time_range, tag)
|
685
|
+
|
686
|
+
# Use operational session for EC2 and resource operations
|
687
|
+
ops_session = _create_operational_session(profile)
|
688
|
+
|
689
|
+
if user_regions:
|
690
|
+
profile_regions = user_regions
|
691
|
+
else:
|
692
|
+
profile_regions = get_accessible_regions(ops_session)
|
693
|
+
|
694
|
+
ec2_data = ec2_summary(ops_session, profile_regions)
|
695
|
+
service_costs, service_cost_data = process_service_costs(cost_data)
|
696
|
+
budget_info = format_budget_info(cost_data["budgets"])
|
697
|
+
account_id = cost_data.get("account_id", "Unknown") or "Unknown"
|
698
|
+
ec2_summary_text = format_ec2_summary(ec2_data)
|
699
|
+
percent_change_in_total_cost = change_in_total_cost(cost_data["current_month"], cost_data["last_month"])
|
700
|
+
|
701
|
+
return {
|
702
|
+
"profile": profile,
|
703
|
+
"account_id": account_id,
|
704
|
+
"last_month": cost_data["last_month"],
|
705
|
+
"current_month": cost_data["current_month"],
|
706
|
+
"service_costs": service_cost_data,
|
707
|
+
"service_costs_formatted": service_costs,
|
708
|
+
"budget_info": budget_info,
|
709
|
+
"ec2_summary": ec2_data,
|
710
|
+
"ec2_summary_formatted": ec2_summary_text,
|
711
|
+
"success": True,
|
712
|
+
"error": None,
|
713
|
+
"current_period_name": cost_data["current_period_name"],
|
714
|
+
"previous_period_name": cost_data["previous_period_name"],
|
715
|
+
"percent_change_in_total_cost": percent_change_in_total_cost,
|
716
|
+
}
|
717
|
+
|
718
|
+
except Exception as e:
|
719
|
+
console.log(f"[red]Error processing profile {profile}: {str(e)}[/]")
|
720
|
+
return {
|
721
|
+
"profile": profile,
|
722
|
+
"account_id": "Error",
|
723
|
+
"last_month": 0,
|
724
|
+
"current_month": 0,
|
725
|
+
"service_costs": [],
|
726
|
+
"service_costs_formatted": [f"Failed to process profile: {str(e)}"],
|
727
|
+
"budget_info": ["N/A"],
|
728
|
+
"ec2_summary": {"N/A": 0},
|
729
|
+
"ec2_summary_formatted": ["Error"],
|
730
|
+
"success": False,
|
731
|
+
"error": str(e),
|
732
|
+
"current_period_name": "Current month",
|
733
|
+
"previous_period_name": "Last month",
|
734
|
+
"percent_change_in_total_cost": None,
|
735
|
+
}
|
736
|
+
|
737
|
+
|
738
|
+
def _process_combined_profiles_enhanced(
|
739
|
+
account_id: str,
|
740
|
+
profiles: List[str],
|
741
|
+
user_regions: Optional[List[str]] = None,
|
742
|
+
time_range: Optional[int] = None,
|
743
|
+
tag: Optional[List[str]] = None,
|
744
|
+
) -> ProfileData:
|
745
|
+
"""
|
746
|
+
Enhanced combined profile processing with multi-profile session support.
|
747
|
+
Aggregates data from multiple profiles in the same AWS account.
|
748
|
+
"""
|
749
|
+
try:
|
750
|
+
primary_profile = profiles[0]
|
751
|
+
|
752
|
+
# Use billing session for cost data aggregation
|
753
|
+
primary_cost_session = _create_cost_session(primary_profile)
|
754
|
+
# Use operational session for resource data
|
755
|
+
primary_ops_session = _create_operational_session(primary_profile)
|
756
|
+
|
757
|
+
# Get cost data using billing session
|
758
|
+
account_cost_data = get_cost_data(primary_cost_session, time_range, tag)
|
759
|
+
|
760
|
+
if user_regions:
|
761
|
+
profile_regions = user_regions
|
762
|
+
else:
|
763
|
+
profile_regions = get_accessible_regions(primary_ops_session)
|
764
|
+
|
765
|
+
# Aggregate EC2 data from all profiles using operational sessions
|
766
|
+
combined_ec2_data = defaultdict(int)
|
767
|
+
for profile in profiles:
|
768
|
+
try:
|
769
|
+
profile_ops_session = _create_operational_session(profile)
|
770
|
+
profile_ec2_data = ec2_summary(profile_ops_session, profile_regions)
|
771
|
+
for instance_type, count in profile_ec2_data.items():
|
772
|
+
combined_ec2_data[instance_type] += count
|
773
|
+
except Exception as e:
|
774
|
+
console.log(f"[yellow]Warning: Could not get EC2 data for profile {profile}: {str(e)}[/]")
|
775
|
+
|
776
|
+
service_costs, service_cost_data = process_service_costs(account_cost_data)
|
777
|
+
budget_info = format_budget_info(account_cost_data["budgets"])
|
778
|
+
ec2_summary_text = format_ec2_summary(dict(combined_ec2_data))
|
779
|
+
percent_change_in_total_cost = change_in_total_cost(
|
780
|
+
account_cost_data["current_month"], account_cost_data["last_month"]
|
781
|
+
)
|
782
|
+
|
783
|
+
profile_list = ", ".join(profiles)
|
784
|
+
console.log(f"[dim cyan]Combined {len(profiles)} profiles for account {account_id}: {profile_list}[/]")
|
785
|
+
|
786
|
+
return {
|
787
|
+
"profile": f"Combined ({profile_list})",
|
788
|
+
"account_id": account_id,
|
789
|
+
"last_month": account_cost_data["last_month"],
|
790
|
+
"current_month": account_cost_data["current_month"],
|
791
|
+
"service_costs": service_cost_data,
|
792
|
+
"service_costs_formatted": service_costs,
|
793
|
+
"budget_info": budget_info,
|
794
|
+
"ec2_summary": dict(combined_ec2_data),
|
795
|
+
"ec2_summary_formatted": ec2_summary_text,
|
796
|
+
"success": True,
|
797
|
+
"error": None,
|
798
|
+
"current_period_name": account_cost_data["current_period_name"],
|
799
|
+
"previous_period_name": account_cost_data["previous_period_name"],
|
800
|
+
"percent_change_in_total_cost": percent_change_in_total_cost,
|
801
|
+
}
|
802
|
+
|
803
|
+
except Exception as e:
|
804
|
+
console.log(f"[red]Error processing combined profiles for account {account_id}: {str(e)}[/]")
|
805
|
+
profile_list = ", ".join(profiles)
|
806
|
+
return {
|
807
|
+
"profile": f"Combined ({profile_list})",
|
808
|
+
"account_id": account_id,
|
809
|
+
"last_month": 0,
|
810
|
+
"current_month": 0,
|
811
|
+
"service_costs": [],
|
812
|
+
"service_costs_formatted": [f"Failed to process combined profiles: {str(e)}"],
|
813
|
+
"budget_info": ["N/A"],
|
814
|
+
"ec2_summary": {"N/A": 0},
|
815
|
+
"ec2_summary_formatted": ["Error"],
|
816
|
+
"success": False,
|
817
|
+
"error": str(e),
|
818
|
+
"current_period_name": "Current month",
|
819
|
+
"previous_period_name": "Last month",
|
820
|
+
"percent_change_in_total_cost": None,
|
821
|
+
}
|
822
|
+
|
823
|
+
|
381
824
|
def _export_dashboard_reports(
|
382
825
|
export_data: List[ProfileData],
|
383
826
|
args: argparse.Namespace,
|
@@ -414,9 +857,37 @@ def _export_dashboard_reports(
|
|
414
857
|
|
415
858
|
|
416
859
|
def run_dashboard(args: argparse.Namespace) -> int:
|
417
|
-
"""Main function to run the AWS FinOps dashboard."""
|
860
|
+
"""Main function to run the AWS FinOps dashboard with multi-profile support."""
|
418
861
|
with Status("[bright_cyan]Initialising...", spinner="aesthetic", speed=0.4):
|
419
862
|
profiles_to_use, user_regions, time_range = _initialize_profiles(args)
|
863
|
+
|
864
|
+
# Display multi-profile configuration at startup
|
865
|
+
billing_profile = os.getenv('BILLING_PROFILE')
|
866
|
+
mgmt_profile = os.getenv('MANAGEMENT_PROFILE')
|
867
|
+
ops_profile = os.getenv('CENTRALISED_OPS_PROFILE')
|
868
|
+
|
869
|
+
if any([billing_profile, mgmt_profile, ops_profile]):
|
870
|
+
console.print("\n[bold bright_cyan]🔧 Multi-Profile Configuration Detected[/]")
|
871
|
+
config_table = Table(
|
872
|
+
title="Profile Configuration",
|
873
|
+
show_header=True,
|
874
|
+
header_style="bold cyan",
|
875
|
+
box=box.SIMPLE,
|
876
|
+
style="dim"
|
877
|
+
)
|
878
|
+
config_table.add_column("Operation Type", style="bold")
|
879
|
+
config_table.add_column("Profile", style="bright_cyan")
|
880
|
+
config_table.add_column("Purpose", style="dim")
|
881
|
+
|
882
|
+
if billing_profile:
|
883
|
+
config_table.add_row("💰 Billing", billing_profile, "Cost Explorer & Budget API access")
|
884
|
+
if mgmt_profile:
|
885
|
+
config_table.add_row("🏛️ Management", mgmt_profile, "Account ID & Organizations operations")
|
886
|
+
if ops_profile:
|
887
|
+
config_table.add_row("⚙️ Operational", ops_profile, "EC2, S3, and resource discovery")
|
888
|
+
|
889
|
+
console.print(config_table)
|
890
|
+
console.print("[dim]Fallback: Using profile-specific sessions when env vars not set[/]\n")
|
420
891
|
|
421
892
|
if args.audit:
|
422
893
|
_run_audit_report(profiles_to_use, args)
|