aws-inventory-manager 0.13.2__py3-none-any.whl

src/config_service/resource_type_mapping.py

@@ -0,0 +1,328 @@
+"""Resource type mapping between AWS Config and collectors.
+
+AWS Config supports ~80+ resource types. This module maps which types
+can be collected via Config vs which must use direct API calls.
+"""
+
+from __future__ import annotations
+
+from typing import Set
+
+# Resource types supported by AWS Config
+# Reference: https://docs.aws.amazon.com/config/latest/developerguide/resource-config-reference.html
+CONFIG_SUPPORTED_TYPES: Set[str] = {
+    # EC2
+    "AWS::EC2::CustomerGateway",
+    "AWS::EC2::EIP",
+    "AWS::EC2::Host",
+    "AWS::EC2::Instance",
+    "AWS::EC2::InternetGateway",
+    "AWS::EC2::NetworkAcl",
+    "AWS::EC2::NetworkInterface",
+    "AWS::EC2::RouteTable",
+    "AWS::EC2::SecurityGroup",
+    "AWS::EC2::Subnet",
+    "AWS::EC2::Volume",
+    "AWS::EC2::VPC",
+    "AWS::EC2::VPNConnection",
+    "AWS::EC2::VPNGateway",
+    "AWS::EC2::NatGateway",
+    "AWS::EC2::EgressOnlyInternetGateway",
+    "AWS::EC2::FlowLog",
+    "AWS::EC2::TransitGateway",
+    "AWS::EC2::TransitGatewayAttachment",
+    "AWS::EC2::TransitGatewayRouteTable",
+    "AWS::EC2::LaunchTemplate",
+    # IAM
+    "AWS::IAM::User",
+    "AWS::IAM::Group",
+    "AWS::IAM::Role",
+    "AWS::IAM::Policy",
+    # S3
+    "AWS::S3::Bucket",
+    "AWS::S3::AccountPublicAccessBlock",
+    # Lambda
+    "AWS::Lambda::Function",
+    "AWS::Lambda::Alias",
+    # RDS
+    "AWS::RDS::DBInstance",
+    "AWS::RDS::DBCluster",
+    "AWS::RDS::DBClusterSnapshot",
+    "AWS::RDS::DBSecurityGroup",
+    "AWS::RDS::DBSnapshot",
+    "AWS::RDS::DBSubnetGroup",
+    "AWS::RDS::EventSubscription",
+    # DynamoDB
+    "AWS::DynamoDB::Table",
+    # CloudWatch
+    "AWS::Logs::LogGroup",
+    "AWS::CloudWatch::Alarm",
+    # SNS
+    "AWS::SNS::Topic",
+    # SQS
+    "AWS::SQS::Queue",
+    # ELB
+    "AWS::ElasticLoadBalancing::LoadBalancer",
+    "AWS::ElasticLoadBalancingV2::LoadBalancer",
+    "AWS::ElasticLoadBalancingV2::TargetGroup",
+    # ECS
+    "AWS::ECS::Cluster",
+    "AWS::ECS::Service",
+    "AWS::ECS::TaskDefinition",
+    # EKS
+    "AWS::EKS::Cluster",
+    "AWS::EKS::FargateProfile",
+    "AWS::EKS::Nodegroup",
+    # KMS
+    "AWS::KMS::Key",
+    # Secrets Manager
+    "AWS::SecretsManager::Secret",
+    # API Gateway
+    "AWS::ApiGateway::RestApi",
+    "AWS::ApiGateway::Stage",
+    "AWS::ApiGatewayV2::Api",
+    "AWS::ApiGatewayV2::Stage",
+    # CloudFormation
+    "AWS::CloudFormation::Stack",
+    # Auto Scaling
+    "AWS::AutoScaling::AutoScalingGroup",
+    "AWS::AutoScaling::LaunchConfiguration",
+    "AWS::AutoScaling::ScalingPolicy",
+    # CloudTrail
+    "AWS::CloudTrail::Trail",
+    # CodeBuild
+    "AWS::CodeBuild::Project",
+    # CodePipeline
+    "AWS::CodePipeline::Pipeline",
+    # Config
+    "AWS::Config::ConfigurationRecorder",
+    "AWS::Config::ConformancePackCompliance",
+    "AWS::Config::ResourceCompliance",
+    # Elasticsearch/OpenSearch
+    "AWS::Elasticsearch::Domain",
+    "AWS::OpenSearch::Domain",
+    # ElastiCache
+    "AWS::ElastiCache::CacheCluster",
+    "AWS::ElastiCache::ReplicationGroup",
+    # EFS
+    "AWS::EFS::FileSystem",
+    "AWS::EFS::AccessPoint",
+    # EventBridge
+    "AWS::Events::Rule",
+    "AWS::Events::EventBus",
+    # Kinesis
+    "AWS::Kinesis::Stream",
+    "AWS::KinesisFirehose::DeliveryStream",
+    # Redshift
+    "AWS::Redshift::Cluster",
+    "AWS::Redshift::ClusterParameterGroup",
+    "AWS::Redshift::ClusterSecurityGroup",
+    "AWS::Redshift::ClusterSubnetGroup",
+    # SSM
+    "AWS::SSM::ManagedInstanceInventory",
+    "AWS::SSM::PatchCompliance",
+    "AWS::SSM::AssociationCompliance",
+    "AWS::SSM::FileData",
+    # Step Functions
+    "AWS::StepFunctions::StateMachine",
+    "AWS::StepFunctions::Activity",
+    # VPC
+    "AWS::EC2::VPCEndpoint",
+    "AWS::EC2::VPCEndpointService",
+    "AWS::EC2::VPCPeeringConnection",
+    # ACM
+    "AWS::ACM::Certificate",
+    # Backup
+    "AWS::Backup::BackupPlan",
+    "AWS::Backup::BackupSelection",
+    "AWS::Backup::BackupVault",
+    "AWS::Backup::RecoveryPoint",
+    # Network Firewall
+    "AWS::NetworkFirewall::Firewall",
+    "AWS::NetworkFirewall::FirewallPolicy",
+    "AWS::NetworkFirewall::RuleGroup",
+    # WAF
+    "AWS::WAF::RateBasedRule",
+    "AWS::WAF::Rule",
+    "AWS::WAF::RuleGroup",
+    "AWS::WAF::WebACL",
+    "AWS::WAFv2::WebACL",
+    "AWS::WAFv2::RuleGroup",
+    "AWS::WAFv2::IPSet",
+    "AWS::WAFv2::RegexPatternSet",
+    "AWS::WAFv2::ManagedRuleSet",
+    # Shield
+    "AWS::Shield::Protection",
+    "AWS::ShieldRegional::Protection",
+    # Systems Manager
+    "AWS::SSM::Parameter",
+    # Route53 (limited support)
+    "AWS::Route53::HostedZone",
+    # Glue
+    "AWS::Glue::Job",
+    "AWS::Glue::Classifier",
+    "AWS::Glue::Crawler",
+    "AWS::Glue::Database",
+    "AWS::Glue::Table",
+    # Athena
+    "AWS::Athena::WorkGroup",
+    "AWS::Athena::DataCatalog",
+    # EMR
+    "AWS::EMR::Cluster",
+    "AWS::EMR::SecurityConfiguration",
+    # SageMaker
+    "AWS::SageMaker::CodeRepository",
+    "AWS::SageMaker::Model",
+    "AWS::SageMaker::NotebookInstance",
+    "AWS::SageMaker::Workteam",
+}
+
+# Resource types that should ALWAYS use direct API collectors
+# (even if technically supported by Config, the direct API is better)
+DIRECT_API_ONLY_TYPES: Set[str] = {
+    # Route53 has better data via direct API
+    "AWS::Route53::HostedZone",
+    # WAF has complex regional/CloudFront distinction
+    "AWS::WAFv2::WebACL::Regional",
+    "AWS::WAFv2::WebACL::CloudFront",
+}
+
+# Mapping from collector service_name to AWS Config resource types
+COLLECTOR_TO_CONFIG_TYPES: dict[str, list[str]] = {
+    "ec2": [
+        "AWS::EC2::Instance",
+        "AWS::EC2::Volume",
+        "AWS::EC2::VPC",
+        "AWS::EC2::SecurityGroup",
+        "AWS::EC2::Subnet",
+        "AWS::EC2::NatGateway",
+        "AWS::EC2::InternetGateway",
+        "AWS::EC2::RouteTable",
+        "AWS::EC2::NetworkAcl",
+        "AWS::EC2::NetworkInterface",
+        "AWS::EC2::EIP",
+    ],
+    "iam": [
+        "AWS::IAM::Role",
+        "AWS::IAM::User",
+        "AWS::IAM::Group",
+        "AWS::IAM::Policy",
+    ],
+    "s3": [
+        "AWS::S3::Bucket",
+    ],
+    "lambda": [
+        "AWS::Lambda::Function",
+    ],
+    "rds": [
+        "AWS::RDS::DBInstance",
+        "AWS::RDS::DBCluster",
+    ],
+    "dynamodb": [
+        "AWS::DynamoDB::Table",
+    ],
+    "cloudwatch": [
+        "AWS::Logs::LogGroup",
+        "AWS::CloudWatch::Alarm",
+    ],
+    "sns": [
+        "AWS::SNS::Topic",
+    ],
+    "sqs": [
+        "AWS::SQS::Queue",
+    ],
+    "elb": [
+        "AWS::ElasticLoadBalancing::LoadBalancer",
+        "AWS::ElasticLoadBalancingV2::LoadBalancer",
+        "AWS::ElasticLoadBalancingV2::TargetGroup",
+    ],
+    "ecs": [
+        "AWS::ECS::Cluster",
+        "AWS::ECS::Service",
+        "AWS::ECS::TaskDefinition",
+    ],
+    "eks": [
+        "AWS::EKS::Cluster",
+        "AWS::EKS::Nodegroup",
+        "AWS::EKS::FargateProfile",
+    ],
+    "kms": [
+        "AWS::KMS::Key",
+    ],
+    "secretsmanager": [
+        "AWS::SecretsManager::Secret",
+    ],
+    "apigateway": [
+        "AWS::ApiGateway::RestApi",
+        "AWS::ApiGateway::Stage",
+        "AWS::ApiGatewayV2::Api",
+    ],
+    "cloudformation": [
+        "AWS::CloudFormation::Stack",
+    ],
+    "codebuild": [
+        "AWS::CodeBuild::Project",
+    ],
+    "codepipeline": [
+        "AWS::CodePipeline::Pipeline",
+    ],
+    "elasticache": [
+        "AWS::ElastiCache::CacheCluster",
+        "AWS::ElastiCache::ReplicationGroup",
+    ],
+    "efs": [
+        "AWS::EFS::FileSystem",
+        "AWS::EFS::AccessPoint",
+    ],
+    "eventbridge": [
+        "AWS::Events::Rule",
+        "AWS::Events::EventBus",
+    ],
+    "stepfunctions": [
+        "AWS::StepFunctions::StateMachine",
+    ],
+    "backup": [
+        "AWS::Backup::BackupPlan",
+        "AWS::Backup::BackupVault",
+    ],
+    "ssm": [
+        "AWS::SSM::Parameter",
+    ],
+    "vpcendpoints": [
+        "AWS::EC2::VPCEndpoint",
+    ],
+    "waf": [
+        "AWS::WAFv2::WebACL",
+        "AWS::WAFv2::RuleGroup",
+        "AWS::WAFv2::IPSet",
+    ],
+    "route53": [
+        "AWS::Route53::HostedZone",
+    ],
+}
+
+
+def is_config_supported_type(resource_type: str) -> bool:
+    """Check if a resource type is supported by AWS Config.
+
+    Args:
+        resource_type: AWS resource type (e.g., "AWS::EC2::Instance")
+
+    Returns:
+        True if the type can be collected via AWS Config
+    """
+    if resource_type in DIRECT_API_ONLY_TYPES:
+        return False
+    return resource_type in CONFIG_SUPPORTED_TYPES
+
+
+def get_config_types_for_service(service_name: str) -> list[str]:
+    """Get AWS Config resource types for a collector service.
+
+    Args:
+        service_name: Collector service name (e.g., "ec2", "s3")
+
+    Returns:
+        List of AWS Config resource types for this service
+    """
+    return COLLECTOR_TO_CONFIG_TYPES.get(service_name, [])

src/cost/__init__.py

@@ -0,0 +1,5 @@
+"""Cost analysis module for tracking baseline vs non-baseline costs."""
+
+from typing import List
+
+__all__: List[str] = []

src/cost/analyzer.py

@@ -0,0 +1,226 @@
+"""Cost analyzer for inventory snapshots."""
+
+import logging
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timedelta
+from typing import Any, Dict, Optional, Set
+
+from ..models.cost_report import CostBreakdown, CostReport
+from ..models.snapshot import Snapshot
+from .explorer import CostExplorerClient
+
+logger = logging.getLogger(__name__)
+
+
+class CostAnalyzer:
+    """Analyze costs for inventory snapshots."""
+
+    def __init__(self, cost_explorer: CostExplorerClient):
+        """Initialize cost analyzer.
+
+        Args:
+            cost_explorer: Cost Explorer client instance
+        """
+        self.cost_explorer = cost_explorer
+
+    def analyze(
+        self,
+        snapshot: Snapshot,
+        start_date: Optional[datetime] = None,
+        end_date: Optional[datetime] = None,
+        granularity: str = "MONTHLY",
+        has_deltas: bool = False,
+        delta_report: Optional[Any] = None,
+    ) -> CostReport:
+        """Analyze costs and separate snapshot resources.
+
+        This implementation uses a simplified heuristic approach:
+        1. Get total costs by service
+        2. Estimate baseline portion based on resource counts in snapshot
+        3. Remaining costs are attributed to non-baseline resources
+
+        Note: For precise cost attribution, AWS would need to provide
+        per-resource cost data, which Cost Explorer doesn't directly expose.
+        This gives a good approximation based on service-level costs.
+
+        Args:
+            snapshot: The baseline snapshot
+            start_date: Start date for cost analysis (default: snapshot date)
+            end_date: End date for cost analysis (default: today)
+            granularity: Cost granularity - DAILY or MONTHLY
+
+        Returns:
+            CostReport with baseline and non-baseline cost breakdown
+        """
+        # Default date range: from snapshot creation to today
+        if not start_date:
+            start_date = snapshot.created_at
+            # Remove timezone for Cost Explorer API (uses dates only, no time)
+            start_date = start_date.replace(tzinfo=None)
+
+        if not end_date:
+            end_date = datetime.now()
+
+        # Ensure both dates are timezone-naive for comparison
+        if hasattr(start_date, "tzinfo") and start_date.tzinfo is not None:
+            start_date = start_date.replace(tzinfo=None)
+        if hasattr(end_date, "tzinfo") and end_date.tzinfo is not None:
+            end_date = end_date.replace(tzinfo=None)
+
+        # Ensure start_date is before end_date
+        # AWS Cost Explorer requires at least 1 day difference
+        if start_date >= end_date:
+            # If dates are the same or inverted, set end_date to start_date + 1 day
+            end_date = start_date + timedelta(days=1)
+
+        logger.debug(f"Analyzing costs from {start_date.strftime('%Y-%m-%d')} " f"to {end_date.strftime('%Y-%m-%d')}")
+
+        # Execute data completeness check and cost retrieval in parallel
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            # Submit both tasks
+            completeness_future = executor.submit(self.cost_explorer.check_data_completeness, end_date)
+            costs_future = executor.submit(self.cost_explorer.get_costs_by_service, start_date, end_date, granularity)
+
+            # Wait for both to complete
+            is_complete, data_through, lag_days = completeness_future.result()
+            service_costs = costs_future.result()
+
+        # If no deltas (no resource changes), ALL costs are baseline
+        if not has_deltas:
+            logger.debug("No resource changes detected - all costs are from snapshot resources")
+            baseline_costs = service_costs.copy()
+            non_baseline_costs: Dict[str, float] = {}
+            baseline_total = sum(baseline_costs.values())
+            non_baseline_total = 0.0
+            total_cost = baseline_total
+            baseline_pct = 100.0
+            non_baseline_pct = 0.0
+        else:
+            # There are deltas - we can't accurately split costs without per-resource pricing
+            # Show total only with a note that we can't split accurately
+            logger.debug("Resource changes detected - showing total costs only")
+            baseline_costs = service_costs.copy()
+            non_baseline_costs = {}
+            baseline_total = sum(baseline_costs.values())
+            non_baseline_total = 0.0
+            total_cost = baseline_total
+            baseline_pct = 100.0
+            non_baseline_pct = 0.0
+            # Note: We could enhance this in the future to track specific resource costs
+            # For now, we show total and list the delta resources separately
+
+        # Create cost breakdowns
+        baseline_breakdown = CostBreakdown(
+            total=baseline_total,
+            by_service=baseline_costs,
+            percentage=baseline_pct,
+        )
+
+        non_baseline_breakdown = CostBreakdown(
+            total=non_baseline_total,
+            by_service=non_baseline_costs,
+            percentage=non_baseline_pct,
+        )
+
+        # Create cost report
+        report = CostReport(
+            generated_at=datetime.now(),
+            baseline_snapshot_name=snapshot.name,
+            period_start=start_date,
+            period_end=end_date,
+            baseline_costs=baseline_breakdown,
+            non_baseline_costs=non_baseline_breakdown,
+            total_cost=total_cost,
+            data_complete=is_complete,
+            data_through=data_through,
+            lag_days=lag_days,
+        )
+
+        logger.info(
+            f"Cost analysis complete: Baseline=${baseline_total:.2f}, "
+            f"Non-baseline=${non_baseline_total:.2f}, Total=${total_cost:.2f}"
+        )
+
+        return report
+
+    def _get_baseline_service_mapping(self, snapshot: Snapshot) -> Set[str]:
+        """Get set of AWS service names that have baseline resources.
+
+        Maps our resource types (e.g., 'AWS::EC2::Instance') to Cost Explorer
+        service names (e.g., 'Amazon Elastic Compute Cloud - Compute').
+
+        Args:
+            snapshot: Baseline snapshot
+
+        Returns:
+            Set of AWS service names from Cost Explorer
+        """
+        # Mapping from our resource types to Cost Explorer service names
+        service_name_map = {
+            "AWS::EC2::Instance": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::Volume": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::VPC": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::SecurityGroup": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::Subnet": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::VPCEndpoint::Interface": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::EC2::VPCEndpoint::Gateway": "Amazon Elastic Compute Cloud - Compute",
+            "AWS::Lambda::Function": "AWS Lambda",
+            "AWS::Lambda::LayerVersion": "AWS Lambda",
+            "AWS::S3::Bucket": "Amazon Simple Storage Service",
+            "AWS::RDS::DBInstance": "Amazon Relational Database Service",
+            "AWS::RDS::DBCluster": "Amazon Relational Database Service",
+            "AWS::IAM::Role": "AWS Identity and Access Management",
+            "AWS::IAM::User": "AWS Identity and Access Management",
+            "AWS::IAM::Policy": "AWS Identity and Access Management",
+            "AWS::IAM::Group": "AWS Identity and Access Management",
+            "AWS::CloudWatch::Alarm": "Amazon CloudWatch",
+            "AWS::CloudWatch::CompositeAlarm": "Amazon CloudWatch",
+            "AWS::Logs::LogGroup": "Amazon CloudWatch",
+            "AWS::SNS::Topic": "Amazon Simple Notification Service",
+            "AWS::SQS::Queue": "Amazon Simple Queue Service",
+            "AWS::DynamoDB::Table": "Amazon DynamoDB",
+            "AWS::ElasticLoadBalancing::LoadBalancer": "Elastic Load Balancing",
+            "AWS::ElasticLoadBalancingV2::LoadBalancer::Application": "Elastic Load Balancing",
+            "AWS::ElasticLoadBalancingV2::LoadBalancer::Network": "Elastic Load Balancing",
+            "AWS::ElasticLoadBalancingV2::LoadBalancer::Gateway": "Elastic Load Balancing",
+            "AWS::CloudFormation::Stack": "AWS CloudFormation",
+            "AWS::ApiGateway::RestApi": "Amazon API Gateway",
+            "AWS::ApiGatewayV2::Api::HTTP": "Amazon API Gateway",
+            "AWS::ApiGatewayV2::Api::WebSocket": "Amazon API Gateway",
+            "AWS::Events::EventBus": "Amazon EventBridge",
+            "AWS::Events::Rule": "Amazon EventBridge",
+            "AWS::SecretsManager::Secret": "AWS Secrets Manager",
+            "AWS::KMS::Key": "AWS Key Management Service",
+            "AWS::SSM::Parameter": "AWS Systems Manager",
+            "AWS::SSM::Document": "AWS Systems Manager",
+            "AWS::Route53::HostedZone": "Amazon Route 53",
+            "AWS::ECS::Cluster": "Amazon EC2 Container Service",
+            "AWS::ECS::Service": "Amazon EC2 Container Service",
+            "AWS::ECS::TaskDefinition": "Amazon EC2 Container Service",
+            "AWS::StepFunctions::StateMachine": "AWS Step Functions",
+            "AWS::WAFv2::WebACL::Regional": "AWS WAF",
+            "AWS::WAFv2::WebACL::CloudFront": "AWS WAF",
+            "AWS::EKS::Cluster": "Amazon Elastic Kubernetes Service",
+            "AWS::EKS::Nodegroup": "Amazon Elastic Kubernetes Service",
+            "AWS::EKS::FargateProfile": "Amazon Elastic Kubernetes Service",
+            "AWS::CodePipeline::Pipeline": "AWS CodePipeline",
+            "AWS::CodeBuild::Project": "AWS CodeBuild",
+            "AWS::Backup::BackupPlan": "AWS Backup",
+            "AWS::Backup::BackupVault": "AWS Backup",
+        }
+
+        baseline_services = set()
+
+        # Get unique resource types from snapshot
+        resource_types = set()
+        for resource in snapshot.resources:
+            resource_types.add(resource.resource_type)
+
+        # Map to Cost Explorer service names
+        for resource_type in resource_types:
+            if resource_type in service_name_map:
+                baseline_services.add(service_name_map[resource_type])
+
+        logger.debug(f"Baseline services: {baseline_services}")
+
+        return baseline_services