stacksage 0.7.2__py3-none-any.whl

stacksage/ai/summarizer.py

@@ -0,0 +1,49 @@
+# stacksage/ai/summarizer.py
+from typing import Any, Dict, List
+
+
+def summarize_analysis(analysis: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Input: full analysis object
+    Output: concise executive summary + top savings drivers
+    """
+    findings: List[Dict[str, Any]] = analysis.get("findings", [])
+    total_savings = float(analysis.get("estimated_monthly_savings", 0) or 0)
+    total_cost = float(analysis.get("estimated_monthly_cost", 0) or 0)
+    # Top savings drivers
+    sorted_findings = sorted(
+        findings,
+        key=lambda f: float(f.get("estimated_monthly_savings_usd", 0) or 0),
+        reverse=True,
+    )
+    top = sorted_findings[:5]
+    drivers = [
+        {
+            "type": f.get("type"),
+            "id": f.get("id"),
+            "region": f.get("region"),
+            "savings": float(f.get("estimated_monthly_savings_usd", 0) or 0),
+            "recommended_action": f.get("recommended_action") or f.get("action"),
+        }
+        for f in top
+    ]
+    summary_text = (
+        (
+            f"Potential monthly savings: ${total_savings:.2f}. "
+            f"Current cost (flagged resources): ${total_cost:.2f}. "
+            f"Primary drivers: "
+            + ", ".join(
+                [f"{d['type']}({d['id']}): ${d['savings']:.2f}" for d in drivers]
+            )
+        )
+        if drivers
+        else (
+            f"Potential monthly savings: ${total_savings:.2f}. "
+            f"Current cost (flagged resources): ${total_cost:.2f}. "
+            "No high-impact drivers identified."
+        )
+    )
+    return {
+        "text": summary_text,
+        "top_drivers": drivers,
+    }

stacksage/analyzer/analysis.py

@@ -0,0 +1,468 @@
+"""
+StackSage Analyzer module
+
+Detects cloud waste and produces findings. Supports optional CloudWatch live-mode
+with a bounded query budget and error provenance. Aggregates CloudWatch datapoints
+as privacy-safe averages only; no object payloads are fetched.
+"""
+
+import json
+import logging
+from typing import Any, Dict, List
+
+from stacksage.analyzer.detectors.architecture import (
+    detect_ecs_to_fargate_migration_opportunities,
+    detect_fargate_spot_opportunities,
+    detect_lambda_graviton_migration_opportunities,
+    detect_rds_to_aurora_serverless_v2,
+    detect_serverless_migration_opportunities,
+)
+from stacksage.analyzer.detectors.cdn import (
+    detect_unused_cloudfront_distributions,
+    detect_unused_route53_hosted_zones,
+)
+from stacksage.analyzer.detectors.cloudwatch import (
+    detect_cloudwatch_logs_retention,
+    detect_overprovisioned_lambda,
+)
+from stacksage.analyzer.detectors.dynamodb import detect_unused_dynamodb_tables
+
+# Import detectors from modular structure
+from stacksage.analyzer.detectors.ebs import (
+    detect_ebs_overprovisioned_performance,
+    detect_gp2_to_gp3_migration,
+    detect_old_snapshots,
+    detect_snapshot_consolidation,
+    detect_unattached_ebs,
+)
+from stacksage.analyzer.detectors.ec2 import (
+    detect_ec2_generation_upgrades,
+    detect_idle_ec2,
+    detect_stopped_ec2,
+)
+from stacksage.analyzer.detectors.elasticache import detect_idle_elasticache_clusters
+from stacksage.analyzer.detectors.guardrails import (
+    detect_anomaly_detection_guardrail,
+    detect_budgets_guardrail,
+)
+from stacksage.analyzer.detectors.network import (
+    detect_idle_elb,
+    detect_lb_empty_target_groups,
+    detect_nat_gateways,
+    detect_unused_eips,
+)
+from stacksage.analyzer.detectors.posture import detect_tier1_posture
+from stacksage.analyzer.detectors.rds import detect_underutilized_rds
+from stacksage.analyzer.detectors.s3 import detect_s3_lifecycle_suggestions
+from stacksage.analyzer.detectors.tagging import detect_untagged_resources
+
+# MetricBudget and cw_avg live in metrics.py to avoid circular imports.
+# Re-exported here so existing callers (tests, cli) keep working.
+from stacksage.analyzer.metrics import MetricBudget, cw_avg  # noqa: F401
+from stacksage.config import StackSageConfig
+
+logger = logging.getLogger(__name__)
+
+# ==============================================================================
+# DETECTORS NOW IMPORTED FROM stacksage.analyzer.detectors.*
+# All detector functions have been moved to modular files for better organization:
+# - EBS detectors → detectors/ebs.py
+# - EC2 detectors → detectors/ec2.py
+# - RDS detectors → detectors/rds.py
+# - Network detectors → detectors/network.py
+# - CloudWatch detectors → detectors/cloudwatch.py
+# - Tagging detector → detectors/tagging.py
+# - S3 detectors → detectors/s3.py
+# ==============================================================================
+
+
+def _apply_config_filters(
+    findings: List[Dict[str, Any]], config: StackSageConfig, raw: Dict[str, Any]
+) -> List[Dict[str, Any]]:
+    """
+    Apply config-based exclusions and filters to findings.
+
+    Filters out findings that match exclusion rules:
+    - Resource ID exclusions
+    - Region exclusions
+    - Detector type exclusions
+    - Tag-based exclusions
+    - Reporting filters (min savings, severity)
+
+    Args:
+        findings: List of raw findings
+        config: StackSageConfig instance
+        raw: Raw scan data for looking up resource tags
+
+    Returns:
+        Filtered list of findings
+    """
+    filtered = []
+
+    for finding in findings:
+        # Check detector type exclusion
+        finding_type = finding.get("type", "")
+        if config.is_detector_excluded(finding_type):
+            continue
+
+        # Check resource ID exclusion
+        resource_id = finding.get("id") or finding.get("resource_id")
+        if resource_id and config.is_resource_excluded(resource_id):
+            continue
+
+        # Check region exclusion
+        region = finding.get("region")
+        if region and config.is_region_excluded(region):
+            continue
+
+        # Check tag-based exclusion (need to look up resource tags from raw data)
+        if _is_finding_excluded_by_tags(finding, config, raw):
+            continue
+
+        # Apply reporting filters (min savings, severity)
+        if not config.should_include_finding(finding):
+            continue
+
+        filtered.append(finding)
+
+    return filtered
+
+
+def _is_finding_excluded_by_tags(
+    finding: Dict[str, Any], config: StackSageConfig, raw: Dict[str, Any]
+) -> bool:
+    """Check if finding should be excluded based on resource tags."""
+    resource_id = finding.get("id") or finding.get("resource_id")
+    resource_type = finding.get("resource_type", "")
+
+    if not resource_id:
+        return False
+
+    # Map resource types to scanner data keys
+    type_mapping = {
+        "ec2": "ec2",
+        "ebs": "ebs",
+        "elb": "load_balancers",
+        "nat": "nat_gateways",
+        "eip": "eips",
+        "rds": "rds",
+        "s3": "s3_buckets",
+        "ecs": "ecs_services",
+    }
+
+    data_key = type_mapping.get(resource_type)
+    if not data_key:
+        return False
+
+    # Find resource in raw data
+    resources = raw.get(data_key, [])
+    for resource in resources:
+        # Match by ID (various ID fields depending on resource type)
+        rid = (
+            resource.get("id")
+            or resource.get("InstanceId")
+            or resource.get("VolumeId")
+            or resource.get("DBInstanceIdentifier")
+            or resource.get("NatGatewayId")
+            or resource.get("AllocationId")
+            or resource.get("Name")
+        )
+
+        if rid == resource_id:
+            tags = resource.get("tags", []) or resource.get("Tags", [])
+            return config.is_excluded_by_tags(tags)
+
+    return False
+
+
+# Main analyzer integration
+def analyze_scan(
+    raw: Dict[str, Any], session=None, options: Dict[str, Any] = None
+) -> Dict[str, Any]:
+    """
+    raw: dict with keys 'ec2','ebs','s3','rds' (as produced by the scanner)
+    session: boto3.Session used for CloudWatch / additional API calls
+    options: dict with optional config_path for stacksage.yml
+    """
+    # Load configuration
+    opts = options or {}
+    config_path = opts.get("config_path")
+    config = StackSageConfig(config_path)
+
+    # When session is None (demo/offline), skip AWS API-dependent detectors
+    offline = session is None
+    use_cw = bool(opts.get("use_cloudwatch") and not offline)
+    lookback_days = int(opts.get("metrics_lookback_days", 14))
+    cw_budget = MetricBudget(int(opts.get("cw_max_queries", 500))) if use_cw else None
+    enable_tier1 = bool(opts.get("enable_tier1_posture", True) and not offline)
+    enable_cost_guardrails = bool(
+        opts.get("enable_cost_guardrails", True) and not offline
+    )
+    regions = opts.get("regions") or []
+    include_bucket_names = bool(opts.get("include_bucket_names", False))
+
+    findings = []
+    # EBS
+    findings.extend(detect_unattached_ebs(raw.get("ebs", [])))
+    # EBS gp2 → gp3 migration opportunities
+    findings.extend(detect_gp2_to_gp3_migration(raw.get("ebs", [])))
+    # EBS performance over-provisioning (CloudWatch enabled)
+    if use_cw:
+        findings.extend(
+            detect_ebs_overprovisioned_performance(
+                session, raw.get("ebs", []), days=lookback_days, budget=cw_budget
+            )
+        )
+    # EBS snapshot consolidation
+    findings.extend(detect_snapshot_consolidation(raw.get("snapshots", [])))
+    # EC2 generation upgrades (t2→t3, m4→m5, etc.) - significant savings opportunities
+    findings.extend(detect_ec2_generation_upgrades(raw.get("ec2", [])))
+    # stopped EC2
+    findings.extend(detect_stopped_ec2(raw.get("ec2", [])))
+    # idle EC2: only when CloudWatch enabled
+    if use_cw:
+        ec2_list = raw.get("ec2") or []
+        findings.extend(
+            detect_idle_ec2(session, ec2_list, days=lookback_days, budget=cw_budget)
+        )
+        # Architecture optimization: serverless migration opportunities (EC2 → Lambda + API Gateway)
+        findings.extend(
+            detect_serverless_migration_opportunities(
+                ec2_list,
+                cw_avg_fn=lambda **kwargs: cw_avg(session, budget=cw_budget, **kwargs),
+                days=lookback_days,
+            )
+        )
+    # Snapshots (from scanner)
+    findings.extend(detect_old_snapshots(raw.get("snapshots", [])))
+    # NAT gateways (offline-safe: uses scanner output only)
+    findings.extend(detect_nat_gateways(raw.get("nat_gateways", [])))
+    # S3 VPC endpoint (REMOVED: speculative, low confidence, no savings data)
+    # findings.extend(detect_missing_s3_endpoint(raw.get('vpc_endpoints', []), list(set(vpc_ids))))
+    # unused EIPs using scanner output
+    findings.extend(detect_unused_eips(raw.get("eips", [])))
+
+    # Load balancer detectors
+    lb_list = raw.get("load_balancers") or []
+    # Empty target groups (offline-safe: uses scanner output only)
+    findings.extend(detect_lb_empty_target_groups(lb_list))
+    # Idle ELB (CloudWatch-based)
+    if use_cw:
+        findings.extend(
+            detect_idle_elb(session, lb_list, days=lookback_days, budget=cw_budget)
+        )
+
+    # RDS underutilization offline skip
+    if use_cw:
+        rds_list = raw.get("rds") or []
+        findings.extend(
+            detect_underutilized_rds(
+                session, rds_list, days=lookback_days, budget=cw_budget
+            )
+        )
+        # Architecture optimization: spiky RDS → Aurora Serverless v2 candidates
+        findings.extend(
+            detect_rds_to_aurora_serverless_v2(
+                session,
+                rds_list,
+                days=lookback_days,
+                budget=cw_budget,
+                cw_avg_func=cw_avg,
+            )
+        )
+    # Lambda over-provisioning (CloudWatch enabled)
+    if use_cw:
+        findings.extend(
+            detect_overprovisioned_lambda(session, days=lookback_days, budget=cw_budget)
+        )
+        # Architecture optimization: Lambda x86_64 → arm64 (Graviton) recommendations
+        findings.extend(
+            detect_lambda_graviton_migration_opportunities(
+                session,
+                regions=(regions if isinstance(regions, list) else None),
+                days=lookback_days,
+                budget=cw_budget,
+                cw_avg_func=cw_avg,
+            )
+        )
+
+        # ECS/Fargate architecture optimization (ECS services inventory required)
+        ecs_services = raw.get("ecs_services") or []
+        findings.extend(
+            detect_ecs_to_fargate_migration_opportunities(
+                session,
+                ecs_services=ecs_services,
+                days=lookback_days,
+                budget=cw_budget,
+                cw_avg_func=cw_avg,
+            )
+        )
+        findings.extend(
+            detect_fargate_spot_opportunities(session, ecs_services=ecs_services)
+        )
+
+    # DynamoDB unused tables (CloudWatch enabled)
+    if use_cw:
+        findings.extend(
+            detect_unused_dynamodb_tables(
+                inventory=raw,
+                cw_avg_fn=lambda **kwargs: cw_avg(session, budget=cw_budget, **kwargs),
+                pricing=None,  # Use internal pricing
+                config=config,
+            )
+        )
+
+    # ElastiCache idle clusters (CloudWatch enabled)
+    if use_cw:
+        findings.extend(
+            detect_idle_elasticache_clusters(
+                inventory=raw,
+                cw_avg_fn=lambda **kwargs: cw_avg(session, budget=cw_budget, **kwargs),
+                pricing=None,
+                config=config,
+            )
+        )
+
+    # CloudFront and Route 53 (CloudWatch enabled, global services)
+    if use_cw:
+        findings.extend(
+            detect_unused_cloudfront_distributions(
+                inventory=raw,
+                cw_avg_fn=lambda **kwargs: cw_avg(session, budget=cw_budget, **kwargs),
+                pricing=None,
+                config=config,
+            )
+        )
+        findings.extend(
+            detect_unused_route53_hosted_zones(
+                inventory=raw,
+                cw_avg_fn=lambda **kwargs: cw_avg(session, budget=cw_budget, **kwargs),
+                pricing=None,
+                config=config,
+            )
+        )
+
+    # CloudWatch Logs retention
+    if use_cw:
+        findings.extend(detect_cloudwatch_logs_retention(session))
+    # S3 lifecycle suggestions (offline-safe: uses bucket metadata only)
+    findings.extend(detect_s3_lifecycle_suggestions(raw.get("s3", [])))
+    # Untagged resources (opt-in only, disabled by default to reduce noise)
+    findings.extend(detect_untagged_resources(raw, options=opts))
+
+    # Cost guardrails (Budgets + Anomaly Detection)
+    if enable_cost_guardrails:
+        findings.extend(
+            detect_budgets_guardrail(session, account_id=opts.get("account_id"))
+        )
+        findings.extend(detect_anomaly_detection_guardrail(session))
+
+    # Tier-1 posture (security + exposure + audit logging)
+    # Appended last to preserve existing finding ordering for CW/cost detectors.
+    if enable_tier1:
+        try:
+            findings.extend(
+                detect_tier1_posture(
+                    session,
+                    raw,
+                    regions=list(regions) if isinstance(regions, list) else [],
+                    include_bucket_names=include_bucket_names,
+                )
+            )
+        except Exception as e:
+            # Never fail the run on posture detector issues.
+            findings.append(
+                {
+                    "type": "posture_detector_error",
+                    "resource_type": "account",
+                    "id": "account",
+                    "region": None,
+                    "severity": "info",
+                    "confidence": 0.5,
+                    "estimated_monthly_savings_usd": 0,
+                    "recommended_action": "review-posture-detectors",
+                    "explanation": f"Posture detectors encountered an error: {e}",
+                }
+            )
+
+    # compute totals
+    # Aggregate totals with resource-level de-duplication to avoid double-counting
+    # Example: the same EBS volume can appear in 'unattached_ebs' and 'gp2_to_gp3_migration'
+    # We count the base monthly cost once per (resource_type,id) and sum savings separately.
+    # Apply config-based exclusions and filters
+    findings = _apply_config_filters(findings, config, raw)
+
+    total_estimated_savings = 0.0
+    cost_per_resource: Dict[str, float] = {}
+    for f in findings:
+        rid = f.get("id") or f.get("resource_id") or f.get("Name")
+        rtype = f.get("resource_type") or f.get("type")
+        key = f"{rtype}:{rid}" if rid and rtype else None
+        savings = float(f.get("estimated_monthly_savings_usd", 0) or 0)
+        cost = float(f.get("estimated_monthly_cost_usd", 0) or 0)
+        total_estimated_savings += savings
+        if key:
+            # Track the maximum cost seen for this resource to avoid double-counting
+            prev = cost_per_resource.get(key, 0.0)
+            cost_per_resource[key] = max(prev, cost)
+    total_estimated_cost = round(sum(cost_per_resource.values()), 2)
+
+    result = {
+        "findings": findings,
+        "estimated_monthly_savings": round(total_estimated_savings, 2),
+        "estimated_monthly_cost": total_estimated_cost,
+        "summary": f"Found {len(findings)} findings. Estimated monthly savings: ${round(total_estimated_savings,2)}",
+        "account_id": opts.get("account_id"),  # Include account_id from options
+        "provenance": {
+            "cloudwatch_metrics": use_cw,
+            "metrics_lookback_days": lookback_days,
+            "cloudwatch_queries_attempted": cw_budget.attempted if cw_budget else None,
+            "cloudwatch_queries_used": cw_budget.used if cw_budget else None,
+            "cloudwatch_queries_remaining": cw_budget.remaining if cw_budget else None,
+            "cloudwatch_query_budget": cw_budget.remaining if cw_budget else None,
+            "cloudwatch_budget_exhausted": (
+                (cw_budget.remaining == 0) if cw_budget else None
+            ),
+            "config_applied": config.config_path is not None,
+        },
+    }
+
+    # Evidence-grade normalization (adds reason_codes + evidence to each finding)
+    from stacksage.analyzer.evidence import normalize_findings
+
+    result["findings"] = normalize_findings(
+        result["findings"],
+        use_cloudwatch=use_cw,
+        lookback_days=lookback_days,
+        cw_provenance=result.get("provenance"),
+    )
+    logger.debug("Analysis summary: %s", result["summary"])
+    return result
+
+
+# If run as main, read reports/scan_raw.json and write findings
+if __name__ == "__main__":
+    import os
+    import sys
+
+    # Enforce license before allowing direct execution
+    try:
+        from stacksage.licensing import LicenseError, enforce_license
+
+        enforce_license()
+    except LicenseError as e:
+        print(f"License error: {e}", file=sys.stderr)
+        sys.exit(2)
+
+    p = os.path.join("reports", "scan_raw.json")
+    if not os.path.exists(p):
+        print("No scan_raw.json found. Run run_scan.py first.")
+        sys.exit(1)
+    raw = json.load(open(p))
+    import boto3
+
+    session = boto3.Session()
+    out = analyze_scan(raw, session=session)
+    with open("reports/findings.json", "w") as f:
+        json.dump(out, f, indent=2)
+    print("Wrote reports/findings.json")

stacksage/analyzer/detectors/__init__.py

@@ -0,0 +1,72 @@
+"""
+StackSage Detectors Module
+
+All cost optimization and waste detection logic organized by resource type.
+"""
+
+# CloudWatch Detectors
+from .cloudwatch import detect_cloudwatch_logs_retention, detect_overprovisioned_lambda
+
+# EBS Detectors
+from .ebs import (
+    detect_ebs_overprovisioned_performance,
+    detect_gp2_to_gp3_migration,
+    detect_old_snapshots,
+    detect_snapshot_consolidation,
+    detect_unattached_ebs,
+)
+
+# EC2 Detectors
+from .ec2 import detect_ec2_generation_upgrades, detect_idle_ec2, detect_stopped_ec2
+
+# Cost guardrails
+from .guardrails import detect_anomaly_detection_guardrail, detect_budgets_guardrail
+
+# Network Detectors
+from .network import (
+    detect_idle_elb,
+    detect_lb_empty_target_groups,
+    detect_missing_s3_endpoint,
+    detect_nat_gateways,
+    detect_unused_eips,
+)
+
+# RDS Detectors
+from .rds import detect_underutilized_rds
+
+# S3 Detectors
+from .s3 import detect_s3_lifecycle_suggestions
+
+# Tagging Detector
+from .tagging import detect_untagged_resources
+
+__all__ = [
+    # EBS
+    "detect_unattached_ebs",
+    "detect_ebs_overprovisioned_performance",
+    "detect_gp2_to_gp3_migration",
+    "detect_old_snapshots",
+    "detect_snapshot_consolidation",
+    # EC2
+    "detect_stopped_ec2",
+    "detect_idle_ec2",
+    "detect_ec2_generation_upgrades",
+    # RDS
+    "detect_underutilized_rds",
+    # Network
+    "detect_unused_eips",
+    "detect_idle_elb",
+    "detect_lb_empty_target_groups",
+    "detect_nat_gateways",
+    "detect_missing_s3_endpoint",
+    # CloudWatch
+    "detect_overprovisioned_lambda",
+    "detect_cloudwatch_logs_retention",
+    # Cost guardrails
+    "detect_budgets_guardrail",
+    "detect_anomaly_detection_guardrail",
+    # Tagging
+    "detect_untagged_resources",
+    # S3
+    "detect_s3_lifecycle_suggestions",
+]