runbooks 0.9.1__py3-none-any.whl → 0.9.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +15 -6
- runbooks/cfat/__init__.py +3 -1
- runbooks/cloudops/__init__.py +3 -1
- runbooks/common/aws_utils.py +367 -0
- runbooks/common/enhanced_logging_example.py +239 -0
- runbooks/common/enhanced_logging_integration_example.py +257 -0
- runbooks/common/logging_integration_helper.py +344 -0
- runbooks/common/profile_utils.py +8 -6
- runbooks/common/rich_utils.py +347 -3
- runbooks/enterprise/logging.py +400 -38
- runbooks/finops/README.md +262 -406
- runbooks/finops/__init__.py +2 -1
- runbooks/finops/accuracy_cross_validator.py +12 -3
- runbooks/finops/commvault_ec2_analysis.py +415 -0
- runbooks/finops/cost_processor.py +718 -42
- runbooks/finops/dashboard_router.py +44 -22
- runbooks/finops/dashboard_runner.py +302 -39
- runbooks/finops/embedded_mcp_validator.py +358 -48
- runbooks/finops/finops_scenarios.py +771 -0
- runbooks/finops/multi_dashboard.py +30 -15
- runbooks/finops/single_dashboard.py +386 -58
- runbooks/finops/types.py +29 -4
- runbooks/inventory/__init__.py +2 -1
- runbooks/main.py +522 -29
- runbooks/operate/__init__.py +3 -1
- runbooks/remediation/__init__.py +3 -1
- runbooks/remediation/commons.py +55 -16
- runbooks/remediation/commvault_ec2_analysis.py +259 -0
- runbooks/remediation/rds_snapshot_list.py +267 -102
- runbooks/remediation/workspaces_list.py +182 -31
- runbooks/security/__init__.py +3 -1
- runbooks/sre/__init__.py +2 -1
- runbooks/utils/__init__.py +81 -6
- runbooks/utils/version_validator.py +241 -0
- runbooks/vpc/__init__.py +2 -1
- runbooks-0.9.4.dist-info/METADATA +563 -0
- {runbooks-0.9.1.dist-info → runbooks-0.9.4.dist-info}/RECORD +41 -38
- {runbooks-0.9.1.dist-info → runbooks-0.9.4.dist-info}/entry_points.txt +1 -0
- runbooks/inventory/cloudtrail.md +0 -727
- runbooks/inventory/discovery.md +0 -81
- runbooks/remediation/CLAUDE.md +0 -100
- runbooks/remediation/DOME9.md +0 -218
- runbooks/security/ENTERPRISE_SECURITY_FRAMEWORK.md +0 -506
- runbooks-0.9.1.dist-info/METADATA +0 -308
- {runbooks-0.9.1.dist-info → runbooks-0.9.4.dist-info}/WHEEL +0 -0
- {runbooks-0.9.1.dist-info → runbooks-0.9.4.dist-info}/licenses/LICENSE +0 -0
- {runbooks-0.9.1.dist-info → runbooks-0.9.4.dist-info}/top_level.txt +0 -0
@@ -13,7 +13,7 @@ from rich.console import Console
|
|
13
13
|
|
14
14
|
from runbooks.finops.aws_client import get_account_id
|
15
15
|
from runbooks.finops.iam_guidance import handle_cost_explorer_error
|
16
|
-
from runbooks.finops.types import BudgetInfo, CostData, EC2Summary, ProfileData
|
16
|
+
from runbooks.finops.types import BudgetInfo, CostData, DualMetricResult, EC2Summary, ProfileData
|
17
17
|
|
18
18
|
console = Console()
|
19
19
|
|
@@ -67,6 +67,241 @@ def filter_analytical_services(
|
|
67
67
|
return filtered_services
|
68
68
|
|
69
69
|
|
70
|
+
class DualMetricCostProcessor:
|
71
|
+
"""Enhanced processor for UnblendedCost (technical) and AmortizedCost (financial) reporting."""
|
72
|
+
|
73
|
+
def __init__(self, session: Session, profile_name: Optional[str] = None):
|
74
|
+
"""Initialize dual-metric cost processor.
|
75
|
+
|
76
|
+
Args:
|
77
|
+
session: AWS boto3 session
|
78
|
+
profile_name: AWS profile name for error handling
|
79
|
+
"""
|
80
|
+
self.session = session
|
81
|
+
self.profile_name = profile_name or "default"
|
82
|
+
self.ce = session.client("ce")
|
83
|
+
|
84
|
+
def collect_dual_metrics(self,
|
85
|
+
account_id: Optional[str] = None,
|
86
|
+
start_date: str = None,
|
87
|
+
end_date: str = None) -> DualMetricResult:
|
88
|
+
"""Collect both UnblendedCost and AmortizedCost for comprehensive reporting.
|
89
|
+
|
90
|
+
Args:
|
91
|
+
account_id: AWS account ID for filtering (multi-account support)
|
92
|
+
start_date: Start date in ISO format (YYYY-MM-DD)
|
93
|
+
end_date: End date in ISO format (YYYY-MM-DD)
|
94
|
+
|
95
|
+
Returns:
|
96
|
+
DualMetricResult with both technical and financial perspectives
|
97
|
+
"""
|
98
|
+
# Build filter for account if provided
|
99
|
+
filter_param = None
|
100
|
+
if account_id:
|
101
|
+
filter_param = {"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}}
|
102
|
+
|
103
|
+
# Set default dates if not provided
|
104
|
+
if not start_date or not end_date:
|
105
|
+
today = date.today()
|
106
|
+
start_date = today.replace(day=1).isoformat()
|
107
|
+
end_date = (today + timedelta(days=1)).isoformat() # AWS CE end date is exclusive
|
108
|
+
|
109
|
+
try:
|
110
|
+
# Technical Analysis (UnblendedCost)
|
111
|
+
console.log("[blue]🔧 Collecting technical cost data (UnblendedCost)[/]")
|
112
|
+
unblended_response = self.ce.get_cost_and_usage(
|
113
|
+
TimePeriod={'Start': start_date, 'End': end_date},
|
114
|
+
Granularity='MONTHLY',
|
115
|
+
Metrics=['UnblendedCost'],
|
116
|
+
GroupBy=[{'Type': 'DIMENSION', 'Key': 'SERVICE'}],
|
117
|
+
**({"Filter": filter_param} if filter_param else {})
|
118
|
+
)
|
119
|
+
|
120
|
+
# Financial Reporting (AmortizedCost)
|
121
|
+
console.log("[blue]📊 Collecting financial cost data (AmortizedCost)[/]")
|
122
|
+
amortized_response = self.ce.get_cost_and_usage(
|
123
|
+
TimePeriod={'Start': start_date, 'End': end_date},
|
124
|
+
Granularity='MONTHLY',
|
125
|
+
Metrics=['AmortizedCost'],
|
126
|
+
GroupBy=[{'Type': 'DIMENSION', 'Key': 'SERVICE'}],
|
127
|
+
**({"Filter": filter_param} if filter_param else {})
|
128
|
+
)
|
129
|
+
|
130
|
+
# Parse UnblendedCost data
|
131
|
+
unblended_costs = {}
|
132
|
+
technical_total = 0.0
|
133
|
+
service_breakdown_unblended = []
|
134
|
+
|
135
|
+
for result in unblended_response.get("ResultsByTime", []):
|
136
|
+
for group in result.get("Groups", []):
|
137
|
+
service = group["Keys"][0]
|
138
|
+
amount = float(group["Metrics"]["UnblendedCost"]["Amount"])
|
139
|
+
if amount > 0.001: # Filter negligible costs
|
140
|
+
unblended_costs[service] = amount
|
141
|
+
technical_total += amount
|
142
|
+
service_breakdown_unblended.append((service, amount))
|
143
|
+
|
144
|
+
# Parse AmortizedCost data
|
145
|
+
amortized_costs = {}
|
146
|
+
financial_total = 0.0
|
147
|
+
service_breakdown_amortized = []
|
148
|
+
|
149
|
+
for result in amortized_response.get("ResultsByTime", []):
|
150
|
+
for group in result.get("Groups", []):
|
151
|
+
service = group["Keys"][0]
|
152
|
+
amount = float(group["Metrics"]["AmortizedCost"]["Amount"])
|
153
|
+
if amount > 0.001: # Filter negligible costs
|
154
|
+
amortized_costs[service] = amount
|
155
|
+
financial_total += amount
|
156
|
+
service_breakdown_amortized.append((service, amount))
|
157
|
+
|
158
|
+
# Calculate variance
|
159
|
+
variance = abs(technical_total - financial_total)
|
160
|
+
variance_percentage = (variance / financial_total * 100) if financial_total > 0 else 0.0
|
161
|
+
|
162
|
+
# Sort service breakdowns by cost (descending)
|
163
|
+
service_breakdown_unblended.sort(key=lambda x: x[1], reverse=True)
|
164
|
+
service_breakdown_amortized.sort(key=lambda x: x[1], reverse=True)
|
165
|
+
|
166
|
+
console.log(f"[green]✅ Dual-metric collection complete: Technical ${technical_total:.2f}, Financial ${financial_total:.2f}[/]")
|
167
|
+
|
168
|
+
return DualMetricResult(
|
169
|
+
unblended_costs=unblended_costs,
|
170
|
+
amortized_costs=amortized_costs,
|
171
|
+
technical_total=technical_total,
|
172
|
+
financial_total=financial_total,
|
173
|
+
variance=variance,
|
174
|
+
variance_percentage=variance_percentage,
|
175
|
+
period_start=start_date,
|
176
|
+
period_end=end_date,
|
177
|
+
service_breakdown_unblended=service_breakdown_unblended,
|
178
|
+
service_breakdown_amortized=service_breakdown_amortized
|
179
|
+
)
|
180
|
+
|
181
|
+
except Exception as e:
|
182
|
+
console.log(f"[red]❌ Dual-metric collection failed: {str(e)}[/]")
|
183
|
+
if "AccessDeniedException" in str(e) and "ce:GetCostAndUsage" in str(e):
|
184
|
+
handle_cost_explorer_error(e, self.profile_name)
|
185
|
+
|
186
|
+
# Return empty result structure
|
187
|
+
return DualMetricResult(
|
188
|
+
unblended_costs={},
|
189
|
+
amortized_costs={},
|
190
|
+
technical_total=0.0,
|
191
|
+
financial_total=0.0,
|
192
|
+
variance=0.0,
|
193
|
+
variance_percentage=0.0,
|
194
|
+
period_start=start_date,
|
195
|
+
period_end=end_date,
|
196
|
+
service_breakdown_unblended=[],
|
197
|
+
service_breakdown_amortized=[]
|
198
|
+
)
|
199
|
+
|
200
|
+
|
201
|
+
def get_equal_period_cost_data(
|
202
|
+
session: Session,
|
203
|
+
profile_name: Optional[str] = None,
|
204
|
+
account_id: Optional[str] = None,
|
205
|
+
months_back: int = 3
|
206
|
+
) -> Dict[str, Any]:
|
207
|
+
"""
|
208
|
+
Get equal-period cost data for accurate trend analysis.
|
209
|
+
|
210
|
+
Addresses the mathematical error where partial current month (e.g., Sept 1-2)
|
211
|
+
was compared against full previous month (Aug 1-31), resulting in misleading trends.
|
212
|
+
|
213
|
+
Args:
|
214
|
+
session: AWS boto3 session
|
215
|
+
profile_name: AWS profile name for error handling
|
216
|
+
account_id: Optional account ID for filtering
|
217
|
+
months_back: Number of complete months to analyze
|
218
|
+
|
219
|
+
Returns:
|
220
|
+
Dict containing monthly cost data with equal periods for accurate trends
|
221
|
+
"""
|
222
|
+
ce = session.client("ce")
|
223
|
+
today = date.today()
|
224
|
+
|
225
|
+
# Calculate complete months for comparison
|
226
|
+
monthly_data = []
|
227
|
+
|
228
|
+
# Get last N complete months (not including current partial month)
|
229
|
+
for i in range(1, months_back + 1): # Start from 1 to skip current month
|
230
|
+
# Calculate the start and end of each complete month
|
231
|
+
if today.month - i > 0:
|
232
|
+
target_month = today.month - i
|
233
|
+
target_year = today.year
|
234
|
+
else:
|
235
|
+
# Handle year boundary
|
236
|
+
target_month = 12 + (today.month - i)
|
237
|
+
target_year = today.year - 1
|
238
|
+
|
239
|
+
# First day of target month
|
240
|
+
month_start = date(target_year, target_month, 1)
|
241
|
+
|
242
|
+
# Last day of target month
|
243
|
+
if target_month == 12:
|
244
|
+
month_end = date(target_year + 1, 1, 1) - timedelta(days=1)
|
245
|
+
else:
|
246
|
+
month_end = date(target_year, target_month + 1, 1) - timedelta(days=1)
|
247
|
+
|
248
|
+
# Build filter for account if provided
|
249
|
+
filter_param = None
|
250
|
+
if account_id:
|
251
|
+
filter_param = {"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}}
|
252
|
+
|
253
|
+
kwargs = {}
|
254
|
+
if filter_param:
|
255
|
+
kwargs["Filter"] = filter_param
|
256
|
+
|
257
|
+
try:
|
258
|
+
response = ce.get_cost_and_usage(
|
259
|
+
TimePeriod={
|
260
|
+
"Start": month_start.isoformat(),
|
261
|
+
"End": (month_end + timedelta(days=1)).isoformat(), # AWS CE end date is exclusive
|
262
|
+
},
|
263
|
+
Granularity="MONTHLY",
|
264
|
+
Metrics=["UnblendedCost"],
|
265
|
+
**kwargs,
|
266
|
+
)
|
267
|
+
|
268
|
+
# Extract cost data
|
269
|
+
total_cost = 0.0
|
270
|
+
for result in response.get("ResultsByTime", []):
|
271
|
+
if "Total" in result and "UnblendedCost" in result["Total"]:
|
272
|
+
total_cost = float(result["Total"]["UnblendedCost"]["Amount"])
|
273
|
+
|
274
|
+
monthly_data.append({
|
275
|
+
"month": month_start.strftime("%b %Y"),
|
276
|
+
"start_date": month_start.isoformat(),
|
277
|
+
"end_date": month_end.isoformat(),
|
278
|
+
"days": (month_end - month_start).days + 1,
|
279
|
+
"cost": total_cost
|
280
|
+
})
|
281
|
+
|
282
|
+
except Exception as e:
|
283
|
+
console.log(f"[yellow]Error getting cost data for {month_start.strftime('%b %Y')}: {e}[/]")
|
284
|
+
if "AccessDeniedException" in str(e) and "ce:GetCostAndUsage" in str(e):
|
285
|
+
from .iam_guidance import handle_cost_explorer_error
|
286
|
+
handle_cost_explorer_error(e, profile_name)
|
287
|
+
|
288
|
+
# Add empty data to maintain structure
|
289
|
+
monthly_data.append({
|
290
|
+
"month": month_start.strftime("%b %Y"),
|
291
|
+
"start_date": month_start.isoformat(),
|
292
|
+
"end_date": month_end.isoformat(),
|
293
|
+
"days": (month_end - month_start).days + 1,
|
294
|
+
"cost": 0.0
|
295
|
+
})
|
296
|
+
|
297
|
+
return {
|
298
|
+
"account_id": get_account_id(session) or "unknown",
|
299
|
+
"monthly_costs": monthly_data,
|
300
|
+
"analysis_type": "equal_period",
|
301
|
+
"profile": session.profile_name or profile_name or "default"
|
302
|
+
}
|
303
|
+
|
304
|
+
|
70
305
|
def get_trend(session: Session, tag: Optional[List[str]] = None, account_id: Optional[str] = None) -> Dict[str, Any]:
|
71
306
|
"""
|
72
307
|
Get cost trend data for an AWS account.
|
@@ -312,16 +547,48 @@ def get_cost_data(
|
|
312
547
|
previous_period_start = previous_period_end - timedelta(days=time_range)
|
313
548
|
|
314
549
|
else:
|
550
|
+
# CRITICAL MATHEMATICAL FIX: Equal period comparisons for accurate trends
|
551
|
+
# Problem: Partial current month vs full previous month = misleading trends
|
552
|
+
# Solution: Same-day comparisons or complete month comparisons
|
553
|
+
|
315
554
|
start_date = today.replace(day=1)
|
316
555
|
end_date = today
|
317
|
-
|
318
|
-
#
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
556
|
+
|
557
|
+
# Detect if we're dealing with a partial month that could cause misleading trends
|
558
|
+
days_into_month = today.day
|
559
|
+
is_partial_month = days_into_month <= 5 # First 5 days are considered "partial"
|
560
|
+
|
561
|
+
if is_partial_month:
|
562
|
+
console.log(f"[yellow]⚠️ Partial month detected ({days_into_month} days into {today.strftime('%B')})[/]")
|
563
|
+
console.log(f"[dim yellow] Trend calculations may show extreme percentages due to limited current data[/]")
|
564
|
+
console.log(f"[dim yellow] Consider using full month comparisons for accurate trend analysis[/]")
|
565
|
+
|
566
|
+
# Current period: start of month to today (include today with +1 day for AWS CE)
|
567
|
+
end_date = today + timedelta(days=1) # AWS Cost Explorer end date is exclusive
|
568
|
+
|
569
|
+
# Previous period: Use same day-of-month from previous month for better comparison
|
570
|
+
# This provides more meaningful trends when current month is partial
|
571
|
+
if is_partial_month and days_into_month > 1:
|
572
|
+
# For partial months, compare same number of days from previous month
|
573
|
+
previous_month_same_day = today.replace(day=1) - timedelta(days=1) # Last day of prev month
|
574
|
+
previous_month_start = previous_month_same_day.replace(day=1)
|
575
|
+
|
576
|
+
# Calculate same day of previous month, handling month boundaries
|
577
|
+
try:
|
578
|
+
previous_month_target_day = previous_month_start.replace(day=today.day)
|
579
|
+
previous_period_start = previous_month_start
|
580
|
+
previous_period_end = previous_month_target_day + timedelta(days=1) # Exclusive end
|
581
|
+
|
582
|
+
console.log(f"[cyan]📊 Using equal-day comparison: {days_into_month} days from current vs previous month[/]")
|
583
|
+
|
584
|
+
except ValueError:
|
585
|
+
# Handle cases where previous month doesn't have the same day (e.g., Feb 30)
|
586
|
+
previous_period_end = previous_month_same_day + timedelta(days=1)
|
587
|
+
previous_period_start = previous_period_end.replace(day=1)
|
588
|
+
else:
|
589
|
+
# Standard full previous month comparison
|
590
|
+
previous_period_end = start_date - timedelta(days=1)
|
591
|
+
previous_period_start = previous_period_end.replace(day=1)
|
325
592
|
|
326
593
|
account_id = get_account_id(session)
|
327
594
|
|
@@ -417,6 +684,20 @@ def get_cost_data(
|
|
417
684
|
if amount > 0.001: # Filter out negligible costs
|
418
685
|
costs_by_service[service] = amount
|
419
686
|
|
687
|
+
# Calculate period metadata for trend context
|
688
|
+
current_period_days = (end_date - start_date).days
|
689
|
+
previous_period_days = (previous_period_end - previous_period_start).days
|
690
|
+
is_partial_comparison = abs(current_period_days - previous_period_days) > 5
|
691
|
+
|
692
|
+
# Enhanced period information for trend analysis
|
693
|
+
period_metadata = {
|
694
|
+
"current_days": current_period_days,
|
695
|
+
"previous_days": previous_period_days,
|
696
|
+
"is_partial_comparison": is_partial_comparison,
|
697
|
+
"comparison_type": "partial_vs_partial" if is_partial_comparison else "equal_periods",
|
698
|
+
"trend_reliability": "low" if is_partial_comparison and abs(current_period_days - previous_period_days) > 10 else "high"
|
699
|
+
}
|
700
|
+
|
420
701
|
return {
|
421
702
|
"account_id": account_id,
|
422
703
|
"current_month": current_period_cost,
|
@@ -432,9 +713,90 @@ def get_cost_data(
|
|
432
713
|
"previous_period_start": previous_period_start.isoformat(),
|
433
714
|
"previous_period_end": previous_period_end.isoformat(),
|
434
715
|
"monthly_costs": None,
|
716
|
+
"period_metadata": period_metadata, # Added for intelligent trend analysis
|
435
717
|
}
|
436
718
|
|
437
719
|
|
720
|
+
def get_quarterly_cost_data(
|
721
|
+
session: Session,
|
722
|
+
profile_name: Optional[str] = None,
|
723
|
+
account_id: Optional[str] = None,
|
724
|
+
) -> Dict[str, float]:
|
725
|
+
"""
|
726
|
+
Get quarterly cost data for enhanced FinOps trend analysis.
|
727
|
+
|
728
|
+
Retrieves cost data for the last complete quarter (3 months) to provide
|
729
|
+
strategic quarterly context for financial planning and trend analysis.
|
730
|
+
|
731
|
+
Args:
|
732
|
+
session: The boto3 session to use
|
733
|
+
profile_name: Optional AWS profile name for enhanced error messaging
|
734
|
+
account_id: Optional account ID to filter costs to specific account
|
735
|
+
|
736
|
+
Returns:
|
737
|
+
Dictionary with service names as keys and quarterly costs as values
|
738
|
+
"""
|
739
|
+
ce = session.client("ce")
|
740
|
+
today = date.today()
|
741
|
+
|
742
|
+
# Calculate last quarter date range
|
743
|
+
# Go back 3 months for quarterly analysis
|
744
|
+
quarterly_end_date = today.replace(day=1) - timedelta(days=1) # Last day of previous month
|
745
|
+
quarterly_start_date = (quarterly_end_date.replace(day=1) - timedelta(days=90)).replace(day=1)
|
746
|
+
|
747
|
+
# Build filters for quarterly analysis
|
748
|
+
filters = []
|
749
|
+
if account_id:
|
750
|
+
account_filter = {"Dimensions": {"Key": "LINKED_ACCOUNT", "Values": [account_id]}}
|
751
|
+
filters.append(account_filter)
|
752
|
+
|
753
|
+
# Combine filters if needed
|
754
|
+
filter_param: Optional[Dict[str, Any]] = None
|
755
|
+
if len(filters) == 1:
|
756
|
+
filter_param = filters[0]
|
757
|
+
elif len(filters) > 1:
|
758
|
+
filter_param = {"And": filters}
|
759
|
+
|
760
|
+
kwargs = {}
|
761
|
+
if filter_param:
|
762
|
+
kwargs["Filter"] = filter_param
|
763
|
+
|
764
|
+
try:
|
765
|
+
quarterly_period_cost_by_service = ce.get_cost_and_usage(
|
766
|
+
TimePeriod={
|
767
|
+
"Start": quarterly_start_date.isoformat(),
|
768
|
+
"End": (quarterly_end_date + timedelta(days=1)).isoformat() # Exclusive end
|
769
|
+
},
|
770
|
+
Granularity="MONTHLY",
|
771
|
+
Metrics=["UnblendedCost"],
|
772
|
+
GroupBy=[{"Type": "DIMENSION", "Key": "SERVICE"}],
|
773
|
+
**kwargs,
|
774
|
+
)
|
775
|
+
except Exception as e:
|
776
|
+
console.log(f"[yellow]Warning: Unable to retrieve quarterly cost data: {e}[/]")
|
777
|
+
if "AccessDeniedException" in str(e) and "ce:GetCostAndUsage" in str(e):
|
778
|
+
handle_cost_explorer_error(e, profile_name)
|
779
|
+
return {}
|
780
|
+
|
781
|
+
# Aggregate quarterly costs by service across the 3-month period
|
782
|
+
quarterly_service_costs: Dict[str, float] = defaultdict(float)
|
783
|
+
|
784
|
+
for result in quarterly_period_cost_by_service.get("ResultsByTime", []):
|
785
|
+
for group in result.get("Groups", []):
|
786
|
+
service = group["Keys"][0]
|
787
|
+
amount = float(group["Metrics"]["UnblendedCost"]["Amount"])
|
788
|
+
quarterly_service_costs[service] += amount
|
789
|
+
|
790
|
+
# Filter out negligible costs and convert to regular dict
|
791
|
+
filtered_quarterly_costs = {}
|
792
|
+
for service, amount in quarterly_service_costs.items():
|
793
|
+
if amount > 0.001: # Filter out negligible costs
|
794
|
+
filtered_quarterly_costs[service] = amount
|
795
|
+
|
796
|
+
console.log(f"[cyan]📊 Retrieved quarterly cost data for {len(filtered_quarterly_costs)} services[/]")
|
797
|
+
return filtered_quarterly_costs
|
798
|
+
|
799
|
+
|
438
800
|
def process_service_costs(
|
439
801
|
cost_data: CostData,
|
440
802
|
) -> Tuple[List[str], List[Tuple[str, float]]]:
|
@@ -461,8 +823,24 @@ def process_service_costs(
|
|
461
823
|
|
462
824
|
|
463
825
|
def format_budget_info(budgets: List[BudgetInfo]) -> List[str]:
|
464
|
-
"""Format budget information for display."""
|
826
|
+
"""Format budget information for display with enhanced error handling."""
|
465
827
|
budget_info: List[str] = []
|
828
|
+
|
829
|
+
# Check if this is an access denied case (common with read-only profiles)
|
830
|
+
if budgets and len(budgets) == 1:
|
831
|
+
first_budget = budgets[0]
|
832
|
+
if isinstance(first_budget, dict):
|
833
|
+
# Check for access denied pattern
|
834
|
+
if first_budget.get('name', '').lower() in ['access denied', 'permission denied', 'n/a']:
|
835
|
+
budget_info.append("ℹ️ Budget data unavailable")
|
836
|
+
budget_info.append("(Read-only profile)")
|
837
|
+
budget_info.append("")
|
838
|
+
budget_info.append("💡 For budget access:")
|
839
|
+
budget_info.append("Add budgets:ViewBudget")
|
840
|
+
budget_info.append("policy to profile")
|
841
|
+
return budget_info
|
842
|
+
|
843
|
+
# Normal budget formatting
|
466
844
|
for budget in budgets:
|
467
845
|
budget_info.append(f"{budget['name']} limit: ${budget['limit']}")
|
468
846
|
budget_info.append(f"{budget['name']} actual: ${budget['actual']:.2f}")
|
@@ -470,23 +848,181 @@ def format_budget_info(budgets: List[BudgetInfo]) -> List[str]:
|
|
470
848
|
budget_info.append(f"{budget['name']} forecast: ${budget['forecast']:.2f}")
|
471
849
|
|
472
850
|
if not budget_info:
|
473
|
-
budget_info.append("No budgets
|
851
|
+
budget_info.append("ℹ️ No budgets configured")
|
852
|
+
budget_info.append("💡 Create a budget to")
|
853
|
+
budget_info.append("track spending limits")
|
474
854
|
|
475
855
|
return budget_info
|
476
856
|
|
477
857
|
|
858
|
+
def calculate_quarterly_enhanced_trend(
|
859
|
+
current: float,
|
860
|
+
previous: float,
|
861
|
+
quarterly: float,
|
862
|
+
current_days: Optional[int] = None,
|
863
|
+
previous_days: Optional[int] = None
|
864
|
+
) -> str:
|
865
|
+
"""
|
866
|
+
Calculate trend with quarterly financial intelligence for strategic decision making.
|
867
|
+
|
868
|
+
Enhanced FinOps trend analysis that combines monthly operational trends with quarterly
|
869
|
+
strategic context to provide executive-ready financial intelligence.
|
870
|
+
|
871
|
+
Args:
|
872
|
+
current: Current period cost
|
873
|
+
previous: Previous period cost
|
874
|
+
quarterly: Last quarter (3-month) average cost
|
875
|
+
current_days: Number of days in current period
|
876
|
+
previous_days: Number of days in previous period
|
877
|
+
|
878
|
+
Returns:
|
879
|
+
Strategic trend indicator with quarterly context
|
880
|
+
"""
|
881
|
+
# Start with existing monthly trend logic
|
882
|
+
monthly_trend = calculate_trend_with_context(current, previous, current_days, previous_days)
|
883
|
+
|
884
|
+
# Add quarterly strategic context if available
|
885
|
+
if quarterly > 0:
|
886
|
+
# Calculate quarterly average for monthly comparison
|
887
|
+
quarterly_monthly_avg = quarterly / 3.0 # 3-month average
|
888
|
+
|
889
|
+
# Compare current month against quarterly average
|
890
|
+
if current > 0:
|
891
|
+
quarterly_variance = ((current - quarterly_monthly_avg) / quarterly_monthly_avg) * 100
|
892
|
+
|
893
|
+
# Strategic quarterly indicators
|
894
|
+
if abs(quarterly_variance) < 10: # Within 10% of quarterly average
|
895
|
+
quarterly_context = "📊" # Consistent with quarterly patterns
|
896
|
+
elif quarterly_variance > 25: # Significantly above quarterly average
|
897
|
+
quarterly_context = "📈" # Above quarterly baseline
|
898
|
+
elif quarterly_variance < -25: # Significantly below quarterly average
|
899
|
+
quarterly_context = "📉" # Below quarterly baseline
|
900
|
+
else:
|
901
|
+
quarterly_context = "📊" # Normal quarterly variation
|
902
|
+
|
903
|
+
# Combine monthly operational trend with quarterly strategic context
|
904
|
+
return f"{quarterly_context} {monthly_trend}"
|
905
|
+
|
906
|
+
# Fallback to standard monthly trend if no quarterly data
|
907
|
+
return monthly_trend
|
908
|
+
|
909
|
+
|
910
|
+
def format_cost_with_precision(amount: float, context: str = "dashboard") -> str:
|
911
|
+
"""
|
912
|
+
Format cost with context-aware precision for consistent display.
|
913
|
+
|
914
|
+
Args:
|
915
|
+
amount: Cost amount to format
|
916
|
+
context: Display context ('executive', 'detailed', 'dashboard')
|
917
|
+
|
918
|
+
Returns:
|
919
|
+
Formatted cost string with appropriate precision
|
920
|
+
"""
|
921
|
+
if context == "executive":
|
922
|
+
# Executive summary - round to nearest dollar for clarity
|
923
|
+
return f"${amount:,.0f}"
|
924
|
+
elif context == "detailed":
|
925
|
+
# Detailed analysis - show full precision
|
926
|
+
return f"${amount:,.2f}"
|
927
|
+
else:
|
928
|
+
# Default dashboard - 2 decimal places
|
929
|
+
return f"${amount:,.2f}"
|
930
|
+
|
931
|
+
|
932
|
+
def calculate_trend_with_context(current: float, previous: float,
|
933
|
+
current_days: Optional[int] = None,
|
934
|
+
previous_days: Optional[int] = None) -> str:
|
935
|
+
"""
|
936
|
+
Calculate trend with statistical context and confidence, handling partial period comparisons.
|
937
|
+
|
938
|
+
CRITICAL MATHEMATICAL FIX: Addresses the business-critical issue where partial current month
|
939
|
+
(e.g., September 1-2: $2.50) was compared against full previous month (August 1-31: $155.00),
|
940
|
+
resulting in misleading -98.4% trend calculations that could cause incorrect business decisions.
|
941
|
+
|
942
|
+
Args:
|
943
|
+
current: Current period cost
|
944
|
+
previous: Previous period cost
|
945
|
+
current_days: Number of days in current period (for partial period detection)
|
946
|
+
previous_days: Number of days in previous period (for partial period detection)
|
947
|
+
|
948
|
+
Returns:
|
949
|
+
Trend string with appropriate context and partial period warnings
|
950
|
+
"""
|
951
|
+
if previous == 0:
|
952
|
+
if current == 0:
|
953
|
+
return "No change (both periods $0)"
|
954
|
+
else:
|
955
|
+
return "New spend (no historical data)"
|
956
|
+
|
957
|
+
# Detect partial period issues
|
958
|
+
partial_period_issue = False
|
959
|
+
if current_days and previous_days:
|
960
|
+
if abs(current_days - previous_days) > 5: # More than 5 days difference
|
961
|
+
partial_period_issue = True
|
962
|
+
|
963
|
+
# Calculate basic percentage change
|
964
|
+
change_percent = ((current - previous) / previous) * 100
|
965
|
+
|
966
|
+
# Handle partial period comparisons with warnings
|
967
|
+
if partial_period_issue:
|
968
|
+
period_info = f" (⚠️ {current_days}d vs {previous_days}d)"
|
969
|
+
if abs(change_percent) > 50:
|
970
|
+
return f"⚠️ Partial data comparison{period_info} - trend not reliable"
|
971
|
+
else:
|
972
|
+
base_trend = f"↑ {change_percent:.1f}%" if change_percent > 0 else f"↓ {abs(change_percent):.1f}%"
|
973
|
+
return f"{base_trend}{period_info}"
|
974
|
+
|
975
|
+
# Standard trend analysis for equal periods
|
976
|
+
if abs(change_percent) > 90:
|
977
|
+
if change_percent > 0:
|
978
|
+
return f"↑ {change_percent:.1f}% (significant increase - verify)"
|
979
|
+
else:
|
980
|
+
return f"↓ {abs(change_percent):.1f}% (significant decrease - verify)"
|
981
|
+
elif abs(change_percent) < 1:
|
982
|
+
return "→ Stable (< 1% change)"
|
983
|
+
else:
|
984
|
+
if change_percent > 0:
|
985
|
+
return f"↑ {change_percent:.1f}%"
|
986
|
+
else:
|
987
|
+
return f"↓ {abs(change_percent):.1f}%"
|
988
|
+
|
989
|
+
|
478
990
|
def format_ec2_summary(ec2_data: EC2Summary) -> List[str]:
|
479
|
-
"""Format EC2 instance summary
|
991
|
+
"""Format EC2 instance summary with enhanced visual hierarchy."""
|
480
992
|
ec2_summary_text: List[str] = []
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
993
|
+
|
994
|
+
# Enhanced state formatting with icons and context
|
995
|
+
state_config = {
|
996
|
+
"running": {"color": "bright_green", "icon": "🟢", "priority": 1},
|
997
|
+
"stopped": {"color": "bright_yellow", "icon": "🟡", "priority": 2},
|
998
|
+
"terminated": {"color": "dim red", "icon": "🔴", "priority": 4},
|
999
|
+
"pending": {"color": "bright_cyan", "icon": "🔵", "priority": 3},
|
1000
|
+
"stopping": {"color": "yellow", "icon": "🟠", "priority": 3}
|
1001
|
+
}
|
1002
|
+
|
1003
|
+
# Sort by priority and then by state name
|
1004
|
+
sorted_states = sorted(
|
1005
|
+
[(state, count) for state, count in ec2_data.items() if count > 0],
|
1006
|
+
key=lambda x: (state_config.get(x[0], {"priority": 99})["priority"], x[0])
|
1007
|
+
)
|
1008
|
+
|
1009
|
+
total_instances = sum(count for _, count in sorted_states)
|
1010
|
+
|
1011
|
+
if sorted_states:
|
1012
|
+
# Header with total count
|
1013
|
+
ec2_summary_text.append(f"[bright_cyan]📊 EC2 Instances ({total_instances} total)[/bright_cyan]")
|
1014
|
+
|
1015
|
+
# Individual states with enhanced styling
|
1016
|
+
for state, count in sorted_states:
|
1017
|
+
config = state_config.get(state, {"color": "white", "icon": "⚪", "priority": 99})
|
1018
|
+
percentage = (count / total_instances * 100) if total_instances > 0 else 0
|
1019
|
+
|
1020
|
+
ec2_summary_text.append(
|
1021
|
+
f" {config['icon']} [{config['color']}]{state.title()}: {count}[/{config['color']}] "
|
1022
|
+
f"[dim]({percentage:.1f}%)[/dim]"
|
485
1023
|
)
|
486
|
-
|
487
|
-
|
488
|
-
if not ec2_summary_text:
|
489
|
-
ec2_summary_text = ["No instances found"]
|
1024
|
+
else:
|
1025
|
+
ec2_summary_text = ["[dim]📭 No EC2 instances found[/dim]"]
|
490
1026
|
|
491
1027
|
return ec2_summary_text
|
492
1028
|
|
@@ -508,6 +1044,7 @@ def export_to_csv(
|
|
508
1044
|
output_dir: Optional[str] = None,
|
509
1045
|
previous_period_dates: str = "N/A",
|
510
1046
|
current_period_dates: str = "N/A",
|
1047
|
+
include_dual_metrics: bool = False,
|
511
1048
|
) -> Optional[str]:
|
512
1049
|
"""Export dashboard data to a CSV file."""
|
513
1050
|
try:
|
@@ -524,37 +1061,129 @@ def export_to_csv(
|
|
524
1061
|
current_period_header = f"Cost for period\n({current_period_dates})"
|
525
1062
|
|
526
1063
|
with open(output_filename, "w", newline="") as csvfile:
|
1064
|
+
# Base fieldnames
|
527
1065
|
fieldnames = [
|
528
1066
|
"CLI Profile",
|
529
1067
|
"AWS Account ID",
|
530
1068
|
previous_period_header,
|
531
1069
|
current_period_header,
|
532
|
-
|
1070
|
+
]
|
1071
|
+
|
1072
|
+
# Add dual-metric columns if requested
|
1073
|
+
if include_dual_metrics:
|
1074
|
+
fieldnames.extend([
|
1075
|
+
f"AmortizedCost {current_period_header}",
|
1076
|
+
f"AmortizedCost {previous_period_header}",
|
1077
|
+
"Metric Variance ($)",
|
1078
|
+
"Metric Variance (%)",
|
1079
|
+
"Cost By Service (UnblendedCost)",
|
1080
|
+
"Cost By Service (AmortizedCost)",
|
1081
|
+
])
|
1082
|
+
else:
|
1083
|
+
fieldnames.append("Cost By Service")
|
1084
|
+
|
1085
|
+
fieldnames.extend([
|
533
1086
|
"Budget Status",
|
534
1087
|
"EC2 Instances",
|
535
|
-
]
|
1088
|
+
])
|
1089
|
+
|
536
1090
|
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
537
1091
|
writer.writeheader()
|
538
1092
|
for row in data:
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
550
|
-
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
1093
|
+
# Enhanced error handling for service costs access
|
1094
|
+
try:
|
1095
|
+
service_costs = row.get("service_costs", [])
|
1096
|
+
if isinstance(service_costs, list) and service_costs:
|
1097
|
+
services_data = "\n".join([f"{service}: ${cost:.2f}" for service, cost in service_costs])
|
1098
|
+
else:
|
1099
|
+
services_data = "No service costs"
|
1100
|
+
except (KeyError, TypeError, AttributeError) as e:
|
1101
|
+
console.print(f"[yellow]Warning: Could not process service costs: {e}[/]")
|
1102
|
+
services_data = "Service costs unavailable"
|
1103
|
+
|
1104
|
+
# Enhanced error handling for budget_info access
|
1105
|
+
try:
|
1106
|
+
budget_info = row.get("budget_info", [])
|
1107
|
+
if isinstance(budget_info, list) and budget_info:
|
1108
|
+
budgets_data = "\n".join(str(item) for item in budget_info)
|
1109
|
+
else:
|
1110
|
+
budgets_data = "No budgets"
|
1111
|
+
except (KeyError, TypeError, AttributeError) as e:
|
1112
|
+
console.print(f"[yellow]Warning: Could not process budget info: {e}[/]")
|
1113
|
+
budgets_data = "Budget info unavailable"
|
1114
|
+
|
1115
|
+
# Enhanced error handling for EC2 summary access
|
1116
|
+
try:
|
1117
|
+
ec2_summary = row.get("ec2_summary", {})
|
1118
|
+
if isinstance(ec2_summary, dict) and ec2_summary:
|
1119
|
+
ec2_data_summary = "\n".join(
|
1120
|
+
[f"{state}: {count}" for state, count in ec2_summary.items() if count > 0]
|
1121
|
+
)
|
1122
|
+
else:
|
1123
|
+
ec2_data_summary = "No EC2 instances"
|
1124
|
+
except (KeyError, TypeError, AttributeError) as e:
|
1125
|
+
console.print(f"[yellow]Warning: Could not process EC2 summary: {e}[/]")
|
1126
|
+
ec2_data_summary = "EC2 summary unavailable"
|
1127
|
+
|
1128
|
+
# Enhanced error handling for writerow with safe field access
|
1129
|
+
try:
|
1130
|
+
# Base row data
|
1131
|
+
row_data = {
|
1132
|
+
"CLI Profile": row.get("profile_name", "Unknown"),
|
1133
|
+
"AWS Account ID": row.get("account_id", "Unknown"),
|
1134
|
+
previous_period_header: row.get("previous_month_formatted", "N/A"),
|
1135
|
+
current_period_header: row.get("current_month_formatted", "N/A"),
|
556
1136
|
}
|
557
|
-
|
1137
|
+
|
1138
|
+
# Add dual-metric data if requested
|
1139
|
+
if include_dual_metrics:
|
1140
|
+
# Calculate variance for dual-metric display
|
1141
|
+
current_unblended = row.get("current_month", 0)
|
1142
|
+
current_amortized = row.get("current_month_amortized", current_unblended)
|
1143
|
+
previous_amortized = row.get("previous_month_amortized", row.get("previous_month", 0))
|
1144
|
+
variance = abs(current_unblended - current_amortized)
|
1145
|
+
variance_pct = (variance / current_amortized * 100) if current_amortized > 0 else 0
|
1146
|
+
|
1147
|
+
# Format amortized service costs
|
1148
|
+
amortized_services_data = "No amortized service costs"
|
1149
|
+
if row.get("service_costs_amortized"):
|
1150
|
+
amortized_services_data = "\n".join([
|
1151
|
+
f"{service}: ${cost:.2f}"
|
1152
|
+
for service, cost in row["service_costs_amortized"]
|
1153
|
+
])
|
1154
|
+
|
1155
|
+
row_data.update({
|
1156
|
+
f"AmortizedCost {current_period_header}": f"${current_amortized:.2f}",
|
1157
|
+
f"AmortizedCost {previous_period_header}": f"${previous_amortized:.2f}",
|
1158
|
+
"Metric Variance ($)": f"${variance:.2f}",
|
1159
|
+
"Metric Variance (%)": f"{variance_pct:.2f}%",
|
1160
|
+
"Cost By Service (UnblendedCost)": services_data or "No costs",
|
1161
|
+
"Cost By Service (AmortizedCost)": amortized_services_data,
|
1162
|
+
})
|
1163
|
+
else:
|
1164
|
+
row_data["Cost By Service"] = services_data or "No costs"
|
1165
|
+
|
1166
|
+
# Add common fields
|
1167
|
+
row_data.update({
|
1168
|
+
"Budget Status": budgets_data or "No budgets",
|
1169
|
+
"EC2 Instances": ec2_data_summary or "No instances",
|
1170
|
+
})
|
1171
|
+
|
1172
|
+
writer.writerow(row_data)
|
1173
|
+
except (KeyError, TypeError) as e:
|
1174
|
+
console.print(f"[yellow]Warning: Could not write CSV row: {e}[/]")
|
1175
|
+
# Write a minimal error row to maintain CSV structure
|
1176
|
+
writer.writerow(
|
1177
|
+
{
|
1178
|
+
"CLI Profile": "Error",
|
1179
|
+
"AWS Account ID": "Error",
|
1180
|
+
previous_period_header: "Error",
|
1181
|
+
current_period_header: "Error",
|
1182
|
+
"Cost By Service": f"Row processing error: {e}",
|
1183
|
+
"Budget Status": "Error",
|
1184
|
+
"EC2 Instances": "Error",
|
1185
|
+
}
|
1186
|
+
)
|
558
1187
|
console.print(f"[bright_green]Exported dashboard data to {os.path.abspath(output_filename)}[/]")
|
559
1188
|
return os.path.abspath(output_filename)
|
560
1189
|
except Exception as e:
|
@@ -562,7 +1191,7 @@ def export_to_csv(
|
|
562
1191
|
return None
|
563
1192
|
|
564
1193
|
|
565
|
-
def export_to_json(data: List[ProfileData], filename: str, output_dir: Optional[str] = None) -> Optional[str]:
|
1194
|
+
def export_to_json(data: List[ProfileData], filename: str, output_dir: Optional[str] = None, include_dual_metrics: bool = False) -> Optional[str]:
|
566
1195
|
"""Export dashboard data to a JSON file."""
|
567
1196
|
try:
|
568
1197
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M")
|
@@ -574,8 +1203,55 @@ def export_to_json(data: List[ProfileData], filename: str, output_dir: Optional[
|
|
574
1203
|
else:
|
575
1204
|
output_filename = base_filename
|
576
1205
|
|
1206
|
+
# Prepare data with dual-metric enhancement if requested
|
1207
|
+
export_data = []
|
1208
|
+
for item in data:
|
1209
|
+
if include_dual_metrics:
|
1210
|
+
# Enhanced data structure for dual metrics
|
1211
|
+
enhanced_item = dict(item) # Copy base data
|
1212
|
+
|
1213
|
+
# Calculate variance metrics
|
1214
|
+
current_unblended = item.get("current_month", 0)
|
1215
|
+
current_amortized = item.get("current_month_amortized", current_unblended)
|
1216
|
+
variance = abs(current_unblended - current_amortized)
|
1217
|
+
variance_pct = (variance / current_amortized * 100) if current_amortized > 0 else 0
|
1218
|
+
|
1219
|
+
# Add dual-metric metadata
|
1220
|
+
enhanced_item.update({
|
1221
|
+
"dual_metric_analysis": {
|
1222
|
+
"unblended_cost": {
|
1223
|
+
"current": current_unblended,
|
1224
|
+
"previous": item.get("previous_month", 0),
|
1225
|
+
"metric_type": "technical",
|
1226
|
+
"description": "UnblendedCost - for DevOps/SRE teams"
|
1227
|
+
},
|
1228
|
+
"amortized_cost": {
|
1229
|
+
"current": current_amortized,
|
1230
|
+
"previous": item.get("previous_month_amortized", item.get("previous_month", 0)),
|
1231
|
+
"metric_type": "financial",
|
1232
|
+
"description": "AmortizedCost - for Finance/Executive teams"
|
1233
|
+
},
|
1234
|
+
"variance_analysis": {
|
1235
|
+
"absolute_variance": variance,
|
1236
|
+
"percentage_variance": variance_pct,
|
1237
|
+
"variance_level": "low" if variance_pct < 1.0 else "moderate" if variance_pct < 5.0 else "high"
|
1238
|
+
}
|
1239
|
+
},
|
1240
|
+
"export_metadata": {
|
1241
|
+
"export_type": "dual_metric",
|
1242
|
+
"export_timestamp": datetime.now().isoformat(),
|
1243
|
+
"metric_explanation": {
|
1244
|
+
"unblended_cost": "Actual costs without Reserved Instance or Savings Plan allocations",
|
1245
|
+
"amortized_cost": "Costs with Reserved Instance and Savings Plan benefits applied"
|
1246
|
+
}
|
1247
|
+
}
|
1248
|
+
})
|
1249
|
+
export_data.append(enhanced_item)
|
1250
|
+
else:
|
1251
|
+
export_data.append(item)
|
1252
|
+
|
577
1253
|
with open(output_filename, "w") as jsonfile:
|
578
|
-
json.dump(
|
1254
|
+
json.dump(export_data, jsonfile, indent=4)
|
579
1255
|
|
580
1256
|
console.print(f"[bright_green]Exported dashboard data to {os.path.abspath(output_filename)}[/]")
|
581
1257
|
return os.path.abspath(output_filename)
|