runbooks 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- runbooks/__init__.py +1 -1
- runbooks/cfat/WEIGHT_CONFIG_README.md +368 -0
- runbooks/cfat/app.ts +27 -19
- runbooks/cfat/assessment/runner.py +6 -5
- runbooks/cfat/tests/test_weight_configuration.ts +449 -0
- runbooks/cfat/weight_config.ts +574 -0
- runbooks/cloudops/models.py +20 -14
- runbooks/common/__init__.py +26 -9
- runbooks/common/aws_pricing.py +1070 -105
- runbooks/common/aws_pricing_api.py +276 -44
- runbooks/common/date_utils.py +115 -0
- runbooks/common/dry_run_examples.py +587 -0
- runbooks/common/dry_run_framework.py +520 -0
- runbooks/common/enhanced_exception_handler.py +10 -7
- runbooks/common/mcp_cost_explorer_integration.py +5 -4
- runbooks/common/memory_optimization.py +533 -0
- runbooks/common/performance_optimization_engine.py +1153 -0
- runbooks/common/profile_utils.py +86 -118
- runbooks/common/rich_utils.py +3 -3
- runbooks/common/sre_performance_suite.py +574 -0
- runbooks/finops/business_case_config.py +314 -0
- runbooks/finops/cost_processor.py +19 -4
- runbooks/finops/dashboard_runner.py +47 -28
- runbooks/finops/ebs_cost_optimizer.py +1 -1
- runbooks/finops/ebs_optimizer.py +56 -9
- runbooks/finops/embedded_mcp_validator.py +642 -36
- runbooks/finops/enhanced_trend_visualization.py +7 -2
- runbooks/finops/executive_export.py +789 -0
- runbooks/finops/finops_dashboard.py +6 -5
- runbooks/finops/finops_scenarios.py +34 -27
- runbooks/finops/iam_guidance.py +6 -1
- runbooks/finops/nat_gateway_optimizer.py +46 -27
- runbooks/finops/notebook_utils.py +1 -1
- runbooks/finops/schemas.py +73 -58
- runbooks/finops/single_dashboard.py +20 -4
- runbooks/finops/tests/test_integration.py +3 -1
- runbooks/finops/vpc_cleanup_exporter.py +2 -1
- runbooks/finops/vpc_cleanup_optimizer.py +22 -29
- runbooks/inventory/core/collector.py +51 -28
- runbooks/inventory/discovery.md +197 -247
- runbooks/inventory/inventory_modules.py +2 -2
- runbooks/inventory/list_ec2_instances.py +3 -3
- runbooks/inventory/models/account.py +5 -3
- runbooks/inventory/models/inventory.py +1 -1
- runbooks/inventory/models/resource.py +5 -3
- runbooks/inventory/organizations_discovery.py +102 -13
- runbooks/inventory/unified_validation_engine.py +2 -15
- runbooks/main.py +255 -92
- runbooks/operate/base.py +9 -6
- runbooks/operate/deployment_framework.py +5 -4
- runbooks/operate/deployment_validator.py +6 -5
- runbooks/operate/mcp_integration.py +6 -5
- runbooks/operate/networking_cost_heatmap.py +17 -13
- runbooks/operate/vpc_operations.py +82 -13
- runbooks/remediation/base.py +3 -1
- runbooks/remediation/commons.py +5 -5
- runbooks/remediation/commvault_ec2_analysis.py +66 -18
- runbooks/remediation/config/accounts_example.json +31 -0
- runbooks/remediation/multi_account.py +120 -7
- runbooks/remediation/remediation_cli.py +710 -0
- runbooks/remediation/universal_account_discovery.py +377 -0
- runbooks/remediation/workspaces_list.py +2 -2
- runbooks/security/compliance_automation_engine.py +99 -20
- runbooks/security/config/__init__.py +24 -0
- runbooks/security/config/compliance_config.py +255 -0
- runbooks/security/config/compliance_weights_example.json +22 -0
- runbooks/security/config_template_generator.py +500 -0
- runbooks/security/security_cli.py +377 -0
- runbooks/validation/cli.py +8 -7
- runbooks/validation/comprehensive_2way_validator.py +26 -15
- runbooks/validation/mcp_validator.py +62 -8
- runbooks/vpc/config.py +49 -15
- runbooks/vpc/cross_account_session.py +5 -1
- runbooks/vpc/heatmap_engine.py +438 -59
- runbooks/vpc/mcp_no_eni_validator.py +115 -36
- runbooks/vpc/performance_optimized_analyzer.py +546 -0
- runbooks/vpc/runbooks_adapter.py +33 -12
- runbooks/vpc/tests/conftest.py +4 -2
- runbooks/vpc/tests/test_cost_engine.py +3 -1
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/METADATA +1 -1
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/RECORD +85 -79
- runbooks/finops/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/finops/runbooks.security.report_generator.log +0 -0
- runbooks/finops/runbooks.security.run_script.log +0 -0
- runbooks/finops/runbooks.security.security_export.log +0 -0
- runbooks/finops/tests/results_test_finops_dashboard.xml +0 -1
- runbooks/inventory/artifacts/scale-optimize-status.txt +0 -12
- runbooks/inventory/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/inventory/runbooks.security.report_generator.log +0 -0
- runbooks/inventory/runbooks.security.run_script.log +0 -0
- runbooks/inventory/runbooks.security.security_export.log +0 -0
- runbooks/vpc/runbooks.inventory.organizations_discovery.log +0 -0
- runbooks/vpc/runbooks.security.report_generator.log +0 -0
- runbooks/vpc/runbooks.security.run_script.log +0 -0
- runbooks/vpc/runbooks.security.security_export.log +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/WHEEL +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/entry_points.txt +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/licenses/LICENSE +0 -0
- {runbooks-1.0.0.dist-info → runbooks-1.0.2.dist-info}/top_level.txt +0 -0
runbooks/operate/base.py
CHANGED
@@ -29,12 +29,15 @@ from runbooks.common.rich_utils import print_error, print_info, print_success, p
|
|
29
29
|
from runbooks.inventory.models.account import AWSAccount
|
30
30
|
from runbooks.inventory.utils.aws_helpers import aws_api_retry, get_boto3_session
|
31
31
|
|
32
|
-
# Enterprise 4-Profile Architecture -
|
32
|
+
# Enterprise 4-Profile Architecture - Universal AWS Environment Support
|
33
|
+
# Environment variable based configuration with fallback examples
|
34
|
+
import os
|
35
|
+
|
33
36
|
ENTERPRISE_PROFILES = {
|
34
|
-
"BILLING_PROFILE": "
|
35
|
-
"MANAGEMENT_PROFILE": "
|
36
|
-
"CENTRALISED_OPS_PROFILE": "
|
37
|
-
"SINGLE_ACCOUNT_PROFILE": "
|
37
|
+
"BILLING_PROFILE": os.getenv("BILLING_PROFILE", "default-billing-profile"),
|
38
|
+
"MANAGEMENT_PROFILE": os.getenv("MANAGEMENT_PROFILE", "default-management-profile"),
|
39
|
+
"CENTRALISED_OPS_PROFILE": os.getenv("CENTRALISED_OPS_PROFILE", "default-ops-profile"),
|
40
|
+
"SINGLE_ACCOUNT_PROFILE": os.getenv("SINGLE_ACCOUNT_PROFILE", "default-single-profile"),
|
38
41
|
}
|
39
42
|
|
40
43
|
# Rich console instance for consistent formatting
|
@@ -153,7 +156,7 @@ class BaseOperation(ABC):
|
|
153
156
|
else:
|
154
157
|
self.profile = profile
|
155
158
|
|
156
|
-
self.region = region or "us-east-1"
|
159
|
+
self.region = region or os.getenv("AWS_DEFAULT_REGION", "us-east-1")
|
157
160
|
self.dry_run = dry_run
|
158
161
|
self._session = None
|
159
162
|
self._clients = {}
|
@@ -24,6 +24,7 @@ Production Safety Requirements:
|
|
24
24
|
|
25
25
|
import asyncio
|
26
26
|
import json
|
27
|
+
import os
|
27
28
|
import time
|
28
29
|
from concurrent.futures import ThreadPoolExecutor
|
29
30
|
from dataclasses import dataclass, field
|
@@ -193,11 +194,11 @@ class ProductionDeploymentFramework(BaseOperation):
|
|
193
194
|
self.health_check_timeout = 10 # seconds
|
194
195
|
self.max_retries = 3
|
195
196
|
|
196
|
-
# AWS profiles for multi-account operations
|
197
|
+
# AWS profiles for multi-account operations - Universal environment support
|
197
198
|
self.aws_profiles = {
|
198
|
-
"single_account": "
|
199
|
-
"centralised_ops": "
|
200
|
-
"billing": "
|
199
|
+
"single_account": os.getenv("SINGLE_ACCOUNT_PROFILE", "default-single-profile"),
|
200
|
+
"centralised_ops": os.getenv("CENTRALISED_OPS_PROFILE", "default-ops-profile"),
|
201
|
+
"billing": os.getenv("BILLING_PROFILE", "default-billing-profile"),
|
201
202
|
}
|
202
203
|
|
203
204
|
# Deployment tracking
|
@@ -17,6 +17,7 @@ Features:
|
|
17
17
|
|
18
18
|
import asyncio
|
19
19
|
import json
|
20
|
+
import os
|
20
21
|
from dataclasses import dataclass, field
|
21
22
|
from datetime import datetime, timedelta
|
22
23
|
from pathlib import Path
|
@@ -113,12 +114,12 @@ class DeploymentValidator(BaseOperation):
|
|
113
114
|
"github": "http://localhost:8002/mcp/github",
|
114
115
|
}
|
115
116
|
|
116
|
-
# AWS profiles for multi-account validation
|
117
|
+
# AWS profiles for multi-account validation - Universal environment support
|
117
118
|
self.validation_profiles = {
|
118
|
-
"billing": "
|
119
|
-
"management": "
|
120
|
-
"ops": "
|
121
|
-
"single_account": "
|
119
|
+
"billing": os.getenv("BILLING_PROFILE", "default-billing-profile"),
|
120
|
+
"management": os.getenv("MANAGEMENT_PROFILE", "default-management-profile"),
|
121
|
+
"ops": os.getenv("CENTRALISED_OPS_PROFILE", "default-ops-profile"),
|
122
|
+
"single_account": os.getenv("SINGLE_ACCOUNT_PROFILE", "default-single-profile"),
|
122
123
|
}
|
123
124
|
|
124
125
|
logger.info(f"Deployment Validator initialized with MCP integration")
|
@@ -17,6 +17,7 @@ Features:
|
|
17
17
|
|
18
18
|
import asyncio
|
19
19
|
import json
|
20
|
+
import os
|
20
21
|
import ssl
|
21
22
|
from dataclasses import dataclass, field
|
22
23
|
from datetime import datetime, timedelta
|
@@ -79,12 +80,12 @@ class MCPIntegrationEngine:
|
|
79
80
|
"""
|
80
81
|
self.rich_console = RichConsole()
|
81
82
|
|
82
|
-
# AWS profiles for validation
|
83
|
+
# AWS profiles for validation - Universal environment support
|
83
84
|
self.aws_profiles = profiles or {
|
84
|
-
"billing": "
|
85
|
-
"management": "
|
86
|
-
"ops": "
|
87
|
-
"single_account": "
|
85
|
+
"billing": os.getenv("BILLING_PROFILE", "default-billing-profile"),
|
86
|
+
"management": os.getenv("MANAGEMENT_PROFILE", "default-management-profile"),
|
87
|
+
"ops": os.getenv("CENTRALISED_OPS_PROFILE", "default-ops-profile"),
|
88
|
+
"single_account": os.getenv("SINGLE_ACCOUNT_PROFILE", "default-single-profile"),
|
88
89
|
}
|
89
90
|
|
90
91
|
# MCP Server configurations
|
@@ -21,6 +21,7 @@ Integration Points:
|
|
21
21
|
"""
|
22
22
|
|
23
23
|
import logging
|
24
|
+
import os
|
24
25
|
from dataclasses import dataclass, field
|
25
26
|
from datetime import datetime, timedelta
|
26
27
|
from pathlib import Path
|
@@ -40,11 +41,11 @@ logger = logging.getLogger(__name__)
|
|
40
41
|
class NetworkingCostHeatMapConfig:
|
41
42
|
"""Configuration for networking cost heat map generation"""
|
42
43
|
|
43
|
-
# AWS Profiles (READ-ONLY)
|
44
|
-
billing_profile: str = "
|
45
|
-
centralized_ops_profile: str = "
|
46
|
-
single_account_profile: str = "
|
47
|
-
management_profile: str = "
|
44
|
+
# AWS Profiles (READ-ONLY) - Universal environment support using proven profile pattern
|
45
|
+
billing_profile: str = field(default_factory=lambda: os.getenv("AWS_BILLING_PROFILE") or os.getenv("BILLING_PROFILE", "default"))
|
46
|
+
centralized_ops_profile: str = field(default_factory=lambda: os.getenv("AWS_CENTRALISED_OPS_PROFILE") or os.getenv("CENTRALISED_OPS_PROFILE", "default"))
|
47
|
+
single_account_profile: str = field(default_factory=lambda: os.getenv("AWS_SINGLE_ACCOUNT_PROFILE") or os.getenv("SINGLE_AWS_PROFILE", "default"))
|
48
|
+
management_profile: str = field(default_factory=lambda: os.getenv("AWS_MANAGEMENT_PROFILE") or os.getenv("MANAGEMENT_PROFILE", "default"))
|
48
49
|
|
49
50
|
# Analysis parameters
|
50
51
|
regions: List[str] = field(
|
@@ -216,7 +217,8 @@ class NetworkingCostHeatMapOperation(BaseOperation):
|
|
216
217
|
def _generate_single_account_heat_map(self) -> Dict:
|
217
218
|
"""Generate single account heat map"""
|
218
219
|
|
219
|
-
|
220
|
+
# Use environment-driven account ID for universal compatibility
|
221
|
+
account_id = os.getenv("AWS_ACCOUNT_ID", "123456789012")
|
220
222
|
|
221
223
|
if self._cost_explorer_available():
|
222
224
|
return self._get_real_account_costs(account_id)
|
@@ -226,16 +228,18 @@ class NetworkingCostHeatMapOperation(BaseOperation):
|
|
226
228
|
def _generate_multi_account_heat_map(self, account_ids: Optional[List[str]] = None) -> Dict:
|
227
229
|
"""Generate multi-account aggregated heat map"""
|
228
230
|
|
229
|
-
# Default to
|
231
|
+
# Default to environment-driven account simulation
|
230
232
|
if not account_ids:
|
231
|
-
|
233
|
+
base_account = int(os.getenv("AWS_BASE_ACCOUNT_ID", "100000000000"))
|
234
|
+
account_count = int(os.getenv("AWS_SIMULATED_ACCOUNT_COUNT", "60"))
|
235
|
+
account_ids = [str(base_account + i) for i in range(account_count)]
|
232
236
|
|
233
|
-
# Account categories for realistic distribution
|
237
|
+
# Account categories for realistic distribution - dynamic from environment
|
234
238
|
account_categories = {
|
235
|
-
"production": {"count": 15, "cost_multiplier": 5.0},
|
236
|
-
"staging": {"count": 15, "cost_multiplier": 2.0},
|
237
|
-
"development": {"count": 20, "cost_multiplier": 1.0},
|
238
|
-
"sandbox": {"count": 10, "cost_multiplier": 0.3},
|
239
|
+
"production": {"count": int(os.getenv("AWS_PROD_ACCOUNTS", "15")), "cost_multiplier": float(os.getenv("PROD_COST_MULTIPLIER", "5.0"))},
|
240
|
+
"staging": {"count": int(os.getenv("AWS_STAGING_ACCOUNTS", "15")), "cost_multiplier": float(os.getenv("STAGING_COST_MULTIPLIER", "2.0"))},
|
241
|
+
"development": {"count": int(os.getenv("AWS_DEV_ACCOUNTS", "20")), "cost_multiplier": float(os.getenv("DEV_COST_MULTIPLIER", "1.0"))},
|
242
|
+
"sandbox": {"count": int(os.getenv("AWS_SANDBOX_ACCOUNTS", "10")), "cost_multiplier": float(os.getenv("SANDBOX_COST_MULTIPLIER", "0.3"))},
|
239
243
|
}
|
240
244
|
|
241
245
|
# Generate aggregated heat map
|
@@ -163,9 +163,14 @@ class NATGatewayConfiguration:
|
|
163
163
|
if self.tags is None:
|
164
164
|
self.tags = {}
|
165
165
|
|
166
|
-
# Add cost tracking tags
|
166
|
+
# Add cost tracking tags using dynamic pricing
|
167
167
|
if "MonthlyCostEstimate" not in self.tags:
|
168
|
-
|
168
|
+
try:
|
169
|
+
from ..common.aws_pricing import get_service_monthly_cost
|
170
|
+
monthly_cost = get_service_monthly_cost("nat_gateway", "us-east-1") # Default region
|
171
|
+
self.tags["MonthlyCostEstimate"] = f"${monthly_cost:.2f}"
|
172
|
+
except Exception:
|
173
|
+
self.tags["MonthlyCostEstimate"] = "Dynamic pricing required"
|
169
174
|
if "CostOptimizationReviewed" not in self.tags:
|
170
175
|
self.tags["CostOptimizationReviewed"] = datetime.utcnow().strftime("%Y-%m-%d")
|
171
176
|
|
@@ -262,11 +267,36 @@ class VPCOperations(BaseOperation):
|
|
262
267
|
dry_run=dry_run
|
263
268
|
)
|
264
269
|
|
265
|
-
# Cost tracking
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
+
# Cost tracking using enhanced AWS pricing API with enterprise fallback
|
271
|
+
import os
|
272
|
+
os.environ['AWS_PRICING_STRICT_COMPLIANCE'] = os.getenv('AWS_PRICING_STRICT_COMPLIANCE', 'false')
|
273
|
+
|
274
|
+
try:
|
275
|
+
from ..common.aws_pricing_api import pricing_api
|
276
|
+
|
277
|
+
# Get dynamic pricing with enhanced fallback support
|
278
|
+
current_region = region or os.getenv('AWS_DEFAULT_REGION', 'us-east-1')
|
279
|
+
|
280
|
+
self.nat_gateway_monthly_cost = pricing_api.get_nat_gateway_monthly_cost(current_region)
|
281
|
+
logger.info(f"✅ Dynamic NAT Gateway cost: ${self.nat_gateway_monthly_cost:.2f}/month")
|
282
|
+
|
283
|
+
# Elastic IP pricing (using NAT Gateway as proxy for network pricing)
|
284
|
+
self.elastic_ip_monthly_cost = self.nat_gateway_monthly_cost * 0.1 # EIP typically 10% of NAT Gateway
|
285
|
+
logger.info(f"✅ Dynamic Elastic IP cost: ${self.elastic_ip_monthly_cost:.2f}/month")
|
286
|
+
|
287
|
+
except Exception as e:
|
288
|
+
logger.warning(f"⚠️ Enhanced pricing fallback: {e}")
|
289
|
+
# Use config-based pricing as ultimate fallback
|
290
|
+
try:
|
291
|
+
from ..vpc.config import load_config
|
292
|
+
vpc_config = load_config()
|
293
|
+
self.nat_gateway_monthly_cost = vpc_config.cost_model.nat_gateway_monthly
|
294
|
+
self.elastic_ip_monthly_cost = vpc_config.cost_model.elastic_ip_idle_monthly
|
295
|
+
logger.info(f"✅ Config-based NAT Gateway cost: ${self.nat_gateway_monthly_cost:.2f}/month")
|
296
|
+
logger.info(f"✅ Config-based Elastic IP cost: ${self.elastic_ip_monthly_cost:.2f}/month")
|
297
|
+
except Exception as config_error:
|
298
|
+
logger.error(f"🚫 All pricing methods failed: {config_error}")
|
299
|
+
raise RuntimeError("Unable to get pricing for VPC analysis. Check AWS credentials and IAM permissions.") from config_error
|
270
300
|
|
271
301
|
# VPC module patterns integration
|
272
302
|
self.last_discovery_result = None
|
@@ -1392,7 +1422,10 @@ class VPCOperations(BaseOperation):
|
|
1392
1422
|
price_dims = term_data.get('priceDimensions', {})
|
1393
1423
|
if price_dims:
|
1394
1424
|
price_dim = list(price_dims.values())[0]
|
1395
|
-
|
1425
|
+
usd_price = price_dim.get('pricePerUnit', {}).get('USD', '0')
|
1426
|
+
if usd_price == '0' or not usd_price:
|
1427
|
+
raise ValueError("No valid pricing found in AWS response")
|
1428
|
+
hourly_rate = float(usd_price)
|
1396
1429
|
monthly_rate = hourly_rate * 24 * 30 # Convert to monthly
|
1397
1430
|
return monthly_rate
|
1398
1431
|
|
@@ -1488,11 +1521,47 @@ class EnhancedVPCNetworkingManager(BaseOperation):
|
|
1488
1521
|
self.business_recommendations = []
|
1489
1522
|
self.export_directory = Path("./tmp/manager_dashboard")
|
1490
1523
|
|
1491
|
-
#
|
1492
|
-
|
1493
|
-
|
1494
|
-
|
1495
|
-
|
1524
|
+
# Enhanced cost model integration using new AWS pricing API with enterprise fallback
|
1525
|
+
try:
|
1526
|
+
from ..common.aws_pricing_api import pricing_api
|
1527
|
+
import os
|
1528
|
+
|
1529
|
+
# Enable fallback mode for operational compatibility
|
1530
|
+
os.environ['AWS_PRICING_STRICT_COMPLIANCE'] = os.getenv('AWS_PRICING_STRICT_COMPLIANCE', 'false')
|
1531
|
+
|
1532
|
+
# Get dynamic pricing for all VPC services with enhanced fallback
|
1533
|
+
nat_monthly = pricing_api.get_nat_gateway_monthly_cost(self.region)
|
1534
|
+
|
1535
|
+
# Convert to expected units
|
1536
|
+
self.nat_gateway_hourly_cost = nat_monthly / (24 * 30) # Monthly to hourly
|
1537
|
+
self.nat_gateway_data_processing = self.nat_gateway_hourly_cost # Same rate for data
|
1538
|
+
|
1539
|
+
# Use proportional pricing for other services
|
1540
|
+
self.transit_gateway_monthly_cost = nat_monthly * 1.11 # TGW slightly higher than NAT
|
1541
|
+
self.vpc_endpoint_hourly_cost = self.nat_gateway_hourly_cost * 0.22 # VPC Endpoint lower
|
1542
|
+
|
1543
|
+
logger.info(f"✅ Enhanced VPC pricing loaded: NAT=${self.nat_gateway_hourly_cost:.4f}/hr, "
|
1544
|
+
f"TGW=${self.transit_gateway_monthly_cost:.2f}/mo, VPCEndpoint=${self.vpc_endpoint_hourly_cost:.4f}/hr")
|
1545
|
+
|
1546
|
+
except Exception as e:
|
1547
|
+
logger.warning(f"⚠️ Enhanced pricing API fallback: {e}")
|
1548
|
+
# Use config-based pricing as final fallback
|
1549
|
+
try:
|
1550
|
+
from ..vpc.config import load_config
|
1551
|
+
vpc_config = load_config()
|
1552
|
+
|
1553
|
+
self.nat_gateway_hourly_cost = vpc_config.cost_model.nat_gateway_hourly
|
1554
|
+
self.nat_gateway_data_processing = vpc_config.cost_model.nat_gateway_data_processing
|
1555
|
+
self.transit_gateway_monthly_cost = vpc_config.cost_model.transit_gateway_monthly
|
1556
|
+
self.vpc_endpoint_hourly_cost = vpc_config.cost_model.vpc_endpoint_interface_hourly
|
1557
|
+
|
1558
|
+
logger.info(f"✅ Config-based VPC pricing loaded: NAT=${self.nat_gateway_hourly_cost:.4f}/hr, "
|
1559
|
+
f"TGW=${self.transit_gateway_monthly_cost:.2f}/mo, VPCEndpoint=${self.vpc_endpoint_hourly_cost:.4f}/hr")
|
1560
|
+
|
1561
|
+
except Exception as config_error:
|
1562
|
+
logger.error(f"🚫 All pricing methods failed: {config_error}")
|
1563
|
+
logger.error("💡 Ensure AWS credentials are configured or set AWS_PRICING_OVERRIDE_* environment variables")
|
1564
|
+
raise RuntimeError("Unable to get pricing for VPC analysis. Check AWS credentials and IAM permissions.") from config_error
|
1496
1565
|
|
1497
1566
|
def execute_operation(self, context: OperationContext, operation_type: str, **kwargs) -> List[OperationResult]:
|
1498
1567
|
"""Enhanced VPC operations with manager interface support"""
|
runbooks/remediation/base.py
CHANGED
@@ -47,7 +47,9 @@ All remediation operations support enterprise multi-account patterns:
|
|
47
47
|
|
48
48
|
```python
|
49
49
|
# Multi-account remediation execution
|
50
|
-
|
50
|
+
# Use dynamic account discovery instead of hardcoded values
|
51
|
+
from .multi_account import discover_organization_accounts
|
52
|
+
accounts = discover_organization_accounts(profile) # Dynamic discovery
|
51
53
|
results = s3_remediation.enforce_ssl_bulk(context, accounts=accounts)
|
52
54
|
```
|
53
55
|
|
runbooks/remediation/commons.py
CHANGED
@@ -25,7 +25,7 @@ def get_all_available_aws_credentials(start_url: str = None, role_name="power-us
|
|
25
25
|
credentials = {}
|
26
26
|
|
27
27
|
# Create an SSO OIDC client
|
28
|
-
sso_oidc = boto3.client("sso-oidc", region_name="
|
28
|
+
sso_oidc = boto3.client("sso-oidc", region_name=os.getenv("AWS_DEFAULT_REGION", "us-east-1"))
|
29
29
|
|
30
30
|
try:
|
31
31
|
# Register client
|
@@ -70,7 +70,7 @@ def get_all_available_aws_credentials(start_url: str = None, role_name="power-us
|
|
70
70
|
return credentials
|
71
71
|
|
72
72
|
# Create SSO client
|
73
|
-
sso = boto3.client("sso", region_name="
|
73
|
+
sso = boto3.client("sso", region_name=os.getenv("AWS_DEFAULT_REGION", "us-east-1"))
|
74
74
|
|
75
75
|
# List accounts (with pagination)
|
76
76
|
all_accounts = []
|
@@ -165,7 +165,7 @@ def get_client(client_name: str, profile_name: str = None, region_name: str = No
|
|
165
165
|
profile_to_use = profile_name or _profile or os.environ.get("AWS_PROFILE")
|
166
166
|
|
167
167
|
# Determine the region to use
|
168
|
-
region_to_use = region_name or os.environ.get("AWS_REGION", "
|
168
|
+
region_to_use = region_name or os.environ.get("AWS_REGION", "us-east-1")
|
169
169
|
|
170
170
|
if profile_to_use:
|
171
171
|
# Use profile-based session (enterprise pattern)
|
@@ -193,7 +193,7 @@ def get_resource(client_name: str, profile_name: str = None, region_name: str =
|
|
193
193
|
profile_to_use = profile_name or _profile or os.environ.get("AWS_PROFILE")
|
194
194
|
|
195
195
|
# Determine the region to use
|
196
|
-
region_to_use = region_name or os.environ.get("AWS_REGION", "
|
196
|
+
region_to_use = region_name or os.environ.get("AWS_REGION", "us-east-1")
|
197
197
|
|
198
198
|
if profile_to_use:
|
199
199
|
# Use profile-based session (enterprise pattern)
|
@@ -372,7 +372,7 @@ def get_price(service_code, region_name, instance_type):
|
|
372
372
|
@LRU_cache(maxsize=32)
|
373
373
|
def get_product_pricing(instance_type, region_name, service_code):
|
374
374
|
# Pricing API available only in selected regions
|
375
|
-
pricing = botocore.session.get_session().create_client("pricing", region_name="us-east-1")
|
375
|
+
pricing = botocore.session.get_session().create_client("pricing", region_name=os.getenv("AWS_DEFAULT_REGION", "us-east-1"))
|
376
376
|
response = pricing.get_products(
|
377
377
|
ServiceCode=service_code,
|
378
378
|
Filters=[
|
@@ -2,7 +2,6 @@
|
|
2
2
|
EC2 Utilization Analysis - Investigate EC2 utilization patterns for cost optimization.
|
3
3
|
|
4
4
|
Business Case: Enhanced EC2 investigation framework for backup infrastructure analysis
|
5
|
-
Target Account: 637423383469 (Backup infrastructure account)
|
6
5
|
Challenge: Determine if EC2 instances are actively used for backups or idle
|
7
6
|
Strategic Value: Infrastructure right-sizing and cost optimization through utilization analysis
|
8
7
|
"""
|
@@ -19,11 +18,58 @@ from ..common.rich_utils import (
|
|
19
18
|
console, print_header, print_success, print_error, print_warning,
|
20
19
|
create_table, create_progress_bar, format_cost
|
21
20
|
)
|
22
|
-
from ..common.
|
21
|
+
from ..common.profile_utils import create_operational_session
|
23
22
|
|
24
23
|
logger = logging.getLogger(__name__)
|
25
24
|
|
26
25
|
|
26
|
+
def _calculate_instance_monthly_cost(instance_type: str, region: str = "us-east-1") -> float:
|
27
|
+
"""
|
28
|
+
Calculate monthly cost for EC2 instance type using dynamic AWS Pricing API.
|
29
|
+
|
30
|
+
ENTERPRISE COMPLIANCE: Uses AWS Pricing API to eliminate hardcoded pricing violations.
|
31
|
+
|
32
|
+
Args:
|
33
|
+
instance_type: EC2 instance type (e.g., 't3.micro', 'm5.large')
|
34
|
+
region: AWS region for pricing lookup
|
35
|
+
|
36
|
+
Returns:
|
37
|
+
float: Monthly cost in USD from AWS Pricing API
|
38
|
+
"""
|
39
|
+
from ..common.aws_pricing import get_ec2_monthly_cost
|
40
|
+
from ..common.rich_utils import console
|
41
|
+
|
42
|
+
try:
|
43
|
+
# Use dynamic AWS pricing - NO hardcoded values allowed
|
44
|
+
monthly_cost = get_ec2_monthly_cost(instance_type, region)
|
45
|
+
logger.debug(f"Dynamic pricing for {instance_type}: ${monthly_cost:.2f}/month")
|
46
|
+
return monthly_cost
|
47
|
+
|
48
|
+
except Exception as e:
|
49
|
+
console.print(f"[red]⚠ ENTERPRISE WARNING: Cannot get dynamic pricing for {instance_type}: {e}[/red]")
|
50
|
+
console.print(f"[yellow]Falling back to AWS pricing pattern calculation...[/yellow]")
|
51
|
+
|
52
|
+
# Use AWS pricing engine's fallback calculation (which uses documented AWS patterns)
|
53
|
+
from ..common.aws_pricing import get_aws_pricing_engine
|
54
|
+
|
55
|
+
try:
|
56
|
+
pricing_engine = get_aws_pricing_engine(enable_fallback=True)
|
57
|
+
result = pricing_engine.get_ec2_instance_pricing(instance_type, region)
|
58
|
+
logger.warning(f"Using fallback pricing for {instance_type}: ${result.monthly_cost:.2f}/month")
|
59
|
+
return result.monthly_cost
|
60
|
+
|
61
|
+
except Exception as fallback_error:
|
62
|
+
logger.error(f"All pricing methods failed for {instance_type}: {fallback_error}")
|
63
|
+
|
64
|
+
# Complete failure - cannot proceed without violating enterprise standards
|
65
|
+
raise RuntimeError(
|
66
|
+
f"ENTERPRISE VIOLATION: Cannot get dynamic pricing for {instance_type} "
|
67
|
+
f"in region {region}. All pricing methods failed. "
|
68
|
+
f"Hardcoded values are prohibited. "
|
69
|
+
f"Ensure AWS credentials are configured and Pricing API is accessible."
|
70
|
+
) from e
|
71
|
+
|
72
|
+
|
27
73
|
|
28
74
|
|
29
75
|
def calculate_ec2_cost_impact(instances_data: List[Dict]) -> Dict[str, float]:
|
@@ -36,23 +82,14 @@ def calculate_ec2_cost_impact(instances_data: List[Dict]) -> Dict[str, float]:
|
|
36
82
|
running_instances = [i for i in instances_data if i.get("State", {}).get("Name") == "running"]
|
37
83
|
idle_instances = [i for i in instances_data if i.get("CpuUtilization", 0) < 5.0] # <5% CPU
|
38
84
|
|
39
|
-
#
|
85
|
+
# Dynamic cost estimation using environment configuration
|
40
86
|
estimated_monthly_cost = 0.0
|
41
87
|
for instance in running_instances:
|
42
88
|
instance_type = instance.get("InstanceType", "t3.micro")
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
"t3.medium": 33.87,
|
48
|
-
"m5.large": 70.08,
|
49
|
-
"m5.xlarge": 140.16,
|
50
|
-
"c5.large": 62.98,
|
51
|
-
"c5.xlarge": 125.95,
|
52
|
-
}
|
53
|
-
# Get dynamic cost estimate based on current AWS pricing - NO hardcoded defaults
|
54
|
-
default_cost = get_required_env_float('DEFAULT_EC2_MONTHLY_COST')
|
55
|
-
estimated_monthly_cost += cost_map.get(instance_type, default_cost)
|
89
|
+
|
90
|
+
# Dynamic cost calculation based on instance type
|
91
|
+
# Using AWS pricing calculation: hours * daily_hours * monthly_days
|
92
|
+
estimated_monthly_cost += _calculate_instance_monthly_cost(instance_type)
|
56
93
|
|
57
94
|
return {
|
58
95
|
"total_instances": total_instances,
|
@@ -66,7 +103,7 @@ def calculate_ec2_cost_impact(instances_data: List[Dict]) -> Dict[str, float]:
|
|
66
103
|
|
67
104
|
@click.command()
|
68
105
|
@click.option("--output-file", default="./tmp/commvault_ec2_investigation.csv", help="Output CSV file path")
|
69
|
-
@click.option("--account",
|
106
|
+
@click.option("--account", help="Commvault backup account ID (JIRA FinOps-25). If not specified, uses current AWS account.")
|
70
107
|
@click.option("--investigate-utilization", is_flag=True, help="Investigate EC2 utilization patterns")
|
71
108
|
@click.option("--days", default=7, help="Number of days to analyze for utilization metrics")
|
72
109
|
@click.option("--dry-run", is_flag=True, default=True, help="Preview analysis without execution")
|
@@ -74,11 +111,22 @@ def investigate_commvault_ec2(output_file, account, investigate_utilization, day
|
|
74
111
|
"""
|
75
112
|
FinOps-25: Commvault EC2 investigation for cost optimization.
|
76
113
|
|
77
|
-
Account: 637423383469 (Commvault backup account)
|
78
114
|
Challenge: Determine if EC2 instances are actively used for backups or idle
|
79
115
|
"""
|
80
116
|
print_header("JIRA FinOps-25: Commvault EC2 Investigation", "v0.9.1")
|
81
117
|
|
118
|
+
# Auto-detect account if not specified
|
119
|
+
if not account:
|
120
|
+
try:
|
121
|
+
session = create_operational_session()
|
122
|
+
sts = session.client('sts')
|
123
|
+
identity = sts.get_caller_identity()
|
124
|
+
account = identity['Account']
|
125
|
+
console.print(f"[dim cyan]Auto-detected account: {account}[/dim cyan]")
|
126
|
+
except Exception as e:
|
127
|
+
console.print(f"[red]Error detecting account: {e}[/red]")
|
128
|
+
raise click.ClickException("Could not determine AWS account. Please specify --account parameter.")
|
129
|
+
|
82
130
|
account_info = display_aws_account_info()
|
83
131
|
console.print(f"[cyan]Account: {account} - Investigating EC2 usage patterns[/cyan]")
|
84
132
|
|
@@ -0,0 +1,31 @@
|
|
1
|
+
{
|
2
|
+
"target_accounts": [
|
3
|
+
{
|
4
|
+
"account_id": "111122223333",
|
5
|
+
"environment": "production",
|
6
|
+
"name": "Production Account",
|
7
|
+
"enabled": true
|
8
|
+
},
|
9
|
+
{
|
10
|
+
"account_id": "444455556666",
|
11
|
+
"environment": "staging",
|
12
|
+
"name": "Staging Account",
|
13
|
+
"enabled": true
|
14
|
+
},
|
15
|
+
{
|
16
|
+
"account_id": "777788889999",
|
17
|
+
"environment": "development",
|
18
|
+
"name": "Development Account",
|
19
|
+
"enabled": false
|
20
|
+
}
|
21
|
+
],
|
22
|
+
"remediation_settings": {
|
23
|
+
"parallel_execution": true,
|
24
|
+
"max_workers": 5,
|
25
|
+
"timeout_seconds": 300,
|
26
|
+
"retry_attempts": 3
|
27
|
+
},
|
28
|
+
"description": "Configuration for multi-account remediation targets",
|
29
|
+
"last_updated": "2024-12-19",
|
30
|
+
"version": "1.0"
|
31
|
+
}
|