aws-inventory-manager 0.2.0__py3-none-any.whl

src/models/inventory.py

@@ -0,0 +1,124 @@
+"""Inventory model for organizing snapshots by account and purpose."""
+
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class Inventory:
+    """Named container for organizing snapshots by account and purpose.
+
+    Attributes:
+        name: Unique identifier within account (alphanumeric + hyphens + underscores, 1-50 chars)
+        account_id: AWS account ID (12 digits)
+        include_tags: Tag filters (resource MUST have ALL)
+        exclude_tags: Tag filters (resource MUST NOT have ANY)
+        snapshots: List of snapshot filenames in this inventory
+        active_snapshot: Filename of active baseline snapshot
+        description: Human-readable description
+        created_at: Inventory creation timestamp (timezone-aware UTC)
+        last_updated: Last modification timestamp (timezone-aware UTC, auto-updated)
+    """
+
+    name: str
+    account_id: str
+    include_tags: Dict[str, str] = field(default_factory=dict)
+    exclude_tags: Dict[str, str] = field(default_factory=dict)
+    snapshots: List[str] = field(default_factory=list)
+    active_snapshot: Optional[str] = None
+    description: str = ""
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    last_updated: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary for YAML storage.
+
+        Returns:
+            Dictionary representation suitable for YAML serialization
+        """
+        return {
+            "name": self.name,
+            "account_id": self.account_id,
+            "description": self.description,
+            "include_tags": self.include_tags,
+            "exclude_tags": self.exclude_tags,
+            "snapshots": self.snapshots,
+            "active_snapshot": self.active_snapshot,
+            "created_at": self.created_at.isoformat(),
+            "last_updated": self.last_updated.isoformat(),
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Inventory":
+        """Deserialize from dictionary (YAML load).
+
+        Args:
+            data: Dictionary loaded from YAML
+
+        Returns:
+            Inventory instance
+        """
+        return cls(
+            name=data["name"],
+            account_id=data["account_id"],
+            description=data.get("description", ""),
+            include_tags=data.get("include_tags", {}),
+            exclude_tags=data.get("exclude_tags", {}),
+            snapshots=data.get("snapshots", []),
+            active_snapshot=data.get("active_snapshot"),
+            created_at=datetime.fromisoformat(data["created_at"]),
+            last_updated=datetime.fromisoformat(data["last_updated"]),
+        )
+
+    def add_snapshot(self, snapshot_filename: str, set_active: bool = False) -> None:
+        """Add snapshot to inventory, optionally marking as active.
+
+        Args:
+            snapshot_filename: Name of snapshot file to add
+            set_active: Whether to mark this snapshot as active baseline
+        """
+        if snapshot_filename not in self.snapshots:
+            self.snapshots.append(snapshot_filename)
+        if set_active:
+            self.active_snapshot = snapshot_filename
+        self.last_updated = datetime.now(timezone.utc)
+
+    def remove_snapshot(self, snapshot_filename: str) -> None:
+        """Remove snapshot from inventory, clearing active if it was active.
+
+        Args:
+            snapshot_filename: Name of snapshot file to remove
+        """
+        if snapshot_filename in self.snapshots:
+            self.snapshots.remove(snapshot_filename)
+        if self.active_snapshot == snapshot_filename:
+            self.active_snapshot = None
+        self.last_updated = datetime.now(timezone.utc)
+
+    def validate(self) -> List[str]:
+        """Validate inventory data, return list of errors.
+
+        Returns:
+            List of validation error messages (empty if valid)
+        """
+        errors = []
+
+        # Validate name format (alphanumeric + hyphens + underscores only)
+        if not self.name or not re.match(r"^[a-zA-Z0-9_-]+$", self.name):
+            errors.append("Name must contain only alphanumeric characters, hyphens, and underscores")
+
+        # Validate name length
+        if len(self.name) > 50:
+            errors.append("Name must be 50 characters or less")
+
+        # Validate account ID format (12 digits)
+        if not self.account_id or not re.match(r"^\d{12}$", self.account_id):
+            errors.append("Account ID must be 12 digits")
+
+        # Validate active snapshot exists in snapshots list
+        if self.active_snapshot and self.active_snapshot not in self.snapshots:
+            errors.append("Active snapshot must exist in snapshots list")
+
+        return errors

src/models/resource.py

@@ -0,0 +1,99 @@
+"""Resource data model representing a single AWS resource."""
+
+import re
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class Resource:
+    """Represents a single AWS resource captured in a snapshot."""
+
+    arn: str
+    resource_type: str
+    name: str
+    region: str
+    config_hash: str
+    raw_config: Dict[str, Any]
+    tags: Dict[str, str] = field(default_factory=dict)
+    created_at: Optional[datetime] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert resource to dictionary for serialization."""
+        return {
+            "arn": self.arn,
+            "type": self.resource_type,
+            "name": self.name,
+            "region": self.region,
+            "tags": self.tags,
+            "config_hash": self.config_hash,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "raw_config": self.raw_config,
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Resource":
+        """Create resource from dictionary."""
+        created_at = None
+        if data.get("created_at"):
+            created_at = datetime.fromisoformat(data["created_at"])
+
+        return cls(
+            arn=data["arn"],
+            resource_type=data["type"],
+            name=data["name"],
+            region=data["region"],
+            config_hash=data["config_hash"],
+            raw_config=data["raw_config"],
+            tags=data.get("tags", {}),
+            created_at=created_at,
+        )
+
+    def validate(self) -> bool:
+        """Validate resource data integrity.
+
+        Returns:
+            True if valid, raises ValueError if invalid
+        """
+        # Validate ARN format
+        arn_pattern = r"^arn:aws:[a-z0-9-]+:[a-z0-9-]*:[0-9]*:.*$"
+        if not re.match(arn_pattern, self.arn):
+            raise ValueError(f"Invalid ARN format: {self.arn}")
+
+        # Validate config_hash is 64-character hex string (SHA256)
+        if not re.match(r"^[a-fA-F0-9]{64}$", self.config_hash):
+            raise ValueError(f"Invalid config_hash: {self.config_hash}. Must be 64-character SHA256 hex string.")
+
+        # Validate region format
+        valid_regions = ["global"] + [
+            "us-east-1",
+            "us-east-2",
+            "us-west-1",
+            "us-west-2",
+            "eu-west-1",
+            "eu-west-2",
+            "eu-west-3",
+            "eu-central-1",
+            "ap-southeast-1",
+            "ap-southeast-2",
+            "ap-northeast-1",
+            "ap-northeast-2",
+            "ca-central-1",
+            "sa-east-1",
+            "ap-south-1",
+        ]
+        # Basic validation - starts with region pattern or is 'global'
+        if self.region != "global" and not any(self.region.startswith(r[:6]) for r in valid_regions if r != "global"):
+            # Allow it anyway - AWS adds new regions regularly
+            pass
+
+        return True
+
+    @property
+    def service(self) -> str:
+        """Extract service name from resource type.
+
+        Example: 'iam:role' -> 'iam'
+        """
+        return self.resource_type.split(":")[0] if ":" in self.resource_type else self.resource_type

src/models/snapshot.py

@@ -0,0 +1,108 @@
+"""Snapshot data model representing a point-in-time inventory of AWS resources."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class Snapshot:
+    """Represents a point-in-time inventory of AWS resources.
+
+    This serves as the baseline reference for delta tracking and cost analysis.
+    """
+
+    name: str
+    created_at: datetime
+    account_id: str
+    regions: List[str]
+    resources: List[Any]  # List[Resource] - avoiding circular import
+    is_active: bool = True
+    resource_count: int = 0
+    service_counts: Dict[str, int] = field(default_factory=dict)
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    filters_applied: Optional[Dict[str, Any]] = None
+    total_resources_before_filter: Optional[int] = None
+    inventory_name: str = "default"  # Name of inventory this snapshot belongs to
+
+    def __post_init__(self) -> None:
+        """Calculate derived fields after initialization."""
+        if self.resource_count == 0:
+            self.resource_count = len(self.resources)
+
+        if not self.service_counts:
+            self._calculate_service_counts()
+
+    def _calculate_service_counts(self) -> None:
+        """Calculate resource counts by service type."""
+        counts: Dict[str, int] = {}
+        for resource in self.resources:
+            service = resource.resource_type.split(":")[0] if ":" in resource.resource_type else resource.resource_type
+            counts[service] = counts.get(service, 0) + 1
+        self.service_counts = counts
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert snapshot to dictionary for serialization."""
+        return {
+            "name": self.name,
+            "created_at": self.created_at.isoformat(),
+            "account_id": self.account_id,
+            "regions": self.regions,
+            "is_active": self.is_active,
+            "resource_count": self.resource_count,
+            "service_counts": self.service_counts,
+            "metadata": self.metadata,
+            "filters_applied": self.filters_applied,
+            "total_resources_before_filter": self.total_resources_before_filter,
+            "inventory_name": self.inventory_name,
+            "resources": [r.to_dict() for r in self.resources],
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Snapshot":
+        """Create snapshot from dictionary.
+
+        Note: This requires Resource class to be imported at call time
+        to avoid circular imports.
+        """
+        from .resource import Resource
+
+        return cls(
+            name=data["name"],
+            created_at=datetime.fromisoformat(data["created_at"]),
+            account_id=data["account_id"],
+            regions=data["regions"],
+            resources=[Resource.from_dict(r) for r in data["resources"]],
+            is_active=data.get("is_active", True),
+            resource_count=data.get("resource_count", 0),
+            service_counts=data.get("service_counts", {}),
+            metadata=data.get("metadata", {}),
+            filters_applied=data.get("filters_applied"),
+            total_resources_before_filter=data.get("total_resources_before_filter"),
+            inventory_name=data.get("inventory_name", "default"),  # Default for backward compatibility
+        )
+
+    def validate(self) -> bool:
+        """Validate snapshot data integrity.
+
+        Returns:
+            True if valid, raises ValueError if invalid
+        """
+        import re
+
+        # Validate name format (alphanumeric, hyphens, underscores)
+        if not re.match(r"^[a-zA-Z0-9_-]+$", self.name):
+            raise ValueError(
+                f"Invalid snapshot name: {self.name}. "
+                f"Must contain only alphanumeric characters, hyphens, and underscores."
+            )
+
+        # Validate account ID (12-digit string)
+        if not re.match(r"^\d{12}$", self.account_id):
+            raise ValueError(f"Invalid AWS account ID: {self.account_id}. Must be a 12-digit string.")
+
+        # Validate regions list is not empty
+        if not self.regions:
+            raise ValueError("Snapshot must include at least one AWS region.")
+
+        return True

src/snapshot/__init__.py

@@ -0,0 +1,6 @@
+"""Snapshot capture and storage functionality."""
+
+from .inventory_storage import InventoryNotFoundError, InventoryStorage
+from .storage import SnapshotStorage
+
+__all__ = ["SnapshotStorage", "InventoryStorage", "InventoryNotFoundError"]

src/snapshot/capturer.py

@@ -0,0 +1,347 @@
+"""Snapshot capture coordinator for AWS resources."""
+
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from threading import Lock
+from typing import TYPE_CHECKING, Dict, List, Optional, Type
+
+import boto3
+from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn
+
+from ..models.snapshot import Snapshot
+
+if TYPE_CHECKING:
+    from .filter import ResourceFilter
+from .resource_collectors.apigateway import APIGatewayCollector
+from .resource_collectors.backup import BackupCollector
+from .resource_collectors.base import BaseResourceCollector
+from .resource_collectors.cloudformation import CloudFormationCollector
+from .resource_collectors.cloudwatch import CloudWatchCollector
+from .resource_collectors.codebuild import CodeBuildCollector
+from .resource_collectors.codepipeline import CodePipelineCollector
+from .resource_collectors.dynamodb import DynamoDBCollector
+from .resource_collectors.ec2 import EC2Collector
+from .resource_collectors.ecs import ECSCollector
+from .resource_collectors.eks import EKSCollector
+from .resource_collectors.elb import ELBCollector
+from .resource_collectors.eventbridge import EventBridgeCollector
+from .resource_collectors.iam import IAMCollector
+from .resource_collectors.kms import KMSCollector
+from .resource_collectors.lambda_func import LambdaCollector
+from .resource_collectors.rds import RDSCollector
+from .resource_collectors.route53 import Route53Collector
+from .resource_collectors.s3 import S3Collector
+from .resource_collectors.secretsmanager import SecretsManagerCollector
+from .resource_collectors.sns import SNSCollector
+from .resource_collectors.sqs import SQSCollector
+from .resource_collectors.ssm import SSMCollector
+from .resource_collectors.stepfunctions import StepFunctionsCollector
+from .resource_collectors.vpcendpoints import VPCEndpointsCollector
+from .resource_collectors.waf import WAFCollector
+
+logger = logging.getLogger(__name__)
+
+
+# Registry of all available collectors
+COLLECTOR_REGISTRY: List[Type[BaseResourceCollector]] = [
+    IAMCollector,
+    LambdaCollector,
+    S3Collector,
+    EC2Collector,
+    RDSCollector,
+    CloudWatchCollector,
+    SNSCollector,
+    SQSCollector,
+    DynamoDBCollector,
+    ELBCollector,
+    CloudFormationCollector,
+    APIGatewayCollector,
+    EventBridgeCollector,
+    SecretsManagerCollector,
+    KMSCollector,
+    SSMCollector,
+    Route53Collector,
+    ECSCollector,
+    StepFunctionsCollector,
+    VPCEndpointsCollector,
+    WAFCollector,
+    EKSCollector,
+    CodePipelineCollector,
+    CodeBuildCollector,
+    BackupCollector,
+]
+
+
+def create_snapshot(
+    name: str,
+    regions: List[str],
+    account_id: str,
+    profile_name: Optional[str] = None,
+    set_active: bool = True,
+    resource_types: Optional[List[str]] = None,
+    parallel_workers: int = 10,
+    resource_filter: Optional["ResourceFilter"] = None,
+    inventory_name: str = "default",
+) -> Snapshot:
+    """Create a comprehensive snapshot of AWS resources.
+
+    Args:
+        name: Snapshot name
+        regions: List of AWS regions to scan
+        account_id: AWS account ID
+        profile_name: AWS profile name (optional)
+        set_active: Whether to set as active baseline
+        resource_types: Optional list of resource types to collect (e.g., ['iam', 'lambda'])
+        parallel_workers: Number of parallel collection tasks
+        resource_filter: Optional ResourceFilter for date/tag-based filtering
+        inventory_name: Name of inventory this snapshot belongs to (default: "default")
+
+    Returns:
+        Snapshot instance with captured resources
+    """
+    logger.debug(f"Creating snapshot '{name}' for regions: {regions}")
+
+    # Create session with optional profile
+    session_kwargs = {}
+    if profile_name:
+        session_kwargs["profile_name"] = profile_name
+
+    session = boto3.Session(**session_kwargs)
+
+    # Collect resources
+    all_resources = []
+    resource_counts = {}  # Track counts per service for progress
+    collection_errors = []  # Track errors for summary
+
+    # Expected errors that we'll suppress (service not enabled, pagination issues, etc.)
+    expected_error_patterns = [
+        "Operation cannot be paginated",
+        "is not subscribed",
+        "AccessDenied",
+        "not authorized",
+        "InvalidAction",
+        "OptInRequired",
+    ]
+
+    def is_expected_error(error_msg: str) -> bool:
+        """Check if error is expected and can be safely ignored."""
+        return any(pattern in error_msg for pattern in expected_error_patterns)
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+    ) as progress:
+        # Determine which collectors to use
+        collectors_to_use = _get_collectors(resource_types)
+
+        # Separate global and regional collectors
+        # Create temporary instances to check is_global_service property
+        global_collectors = []
+        regional_collectors = []
+        for c in collectors_to_use:
+            temp_instance = c(session, "us-east-1")
+            if temp_instance.is_global_service:
+                global_collectors.append(c)
+            else:
+                regional_collectors.append(c)
+
+        total_tasks = len(global_collectors) + (len(regional_collectors) * len(regions))
+        main_task = progress.add_task(
+            f"[bold]Collecting AWS resources from {len(regions)} region(s)...", total=total_tasks
+        )
+
+        # Thread-safe lock for updating shared state
+        lock = Lock()
+
+        def collect_service(collector_class: Type[BaseResourceCollector], region: str, is_global: bool = False) -> Dict:
+            """Collect resources for a single service in a region (thread-safe)."""
+            try:
+                collector = collector_class(session, region)
+                service_name = collector.service_name.upper()
+                region_label = "global" if is_global else region
+
+                # Update progress (thread-safe)
+                with lock:
+                    progress.update(main_task, description=f"📦 {service_name} • {region_label}")
+
+                resources = collector.collect()
+
+                logger.debug(f"Collected {len(resources)} {service_name} resources from {region_label}")
+
+                return {"success": True, "resources": resources, "service": service_name, "region": region_label}
+
+            except Exception as e:
+                error_msg = str(e)
+                service_name = collector_class.__name__.replace("Collector", "").upper()
+                region_label = "global" if is_global else region
+
+                if not is_expected_error(error_msg):
+                    logger.debug(f"Collection error - {service_name} ({region_label}): {error_msg[:80]}")
+                    return {
+                        "success": False,
+                        "error": {"service": service_name, "region": region_label, "error": error_msg[:100]},
+                    }
+                else:
+                    logger.debug(f"Skipping {service_name} in {region_label} (not available): {error_msg[:80]}")
+                    return {"success": False, "expected": True}
+
+        # Create list of collection tasks
+        collection_tasks = []
+
+        # Add global service tasks
+        for collector_class in global_collectors:
+            collection_tasks.append((collector_class, "us-east-1", True))
+
+        # Add regional service tasks
+        for region in regions:
+            for collector_class in regional_collectors:
+                collection_tasks.append((collector_class, region, False))
+
+        # Execute collections in parallel
+        with ThreadPoolExecutor(max_workers=parallel_workers) as executor:
+            # Submit all tasks
+            future_to_task = {
+                executor.submit(collect_service, collector_class, region, is_global): (
+                    collector_class,
+                    region,
+                    is_global,
+                )
+                for collector_class, region, is_global in collection_tasks
+            }
+
+            # Process completed tasks
+            for future in as_completed(future_to_task):
+                result = future.result()
+
+                if result["success"]:
+                    with lock:
+                        all_resources.extend(result["resources"])
+                        if result["region"] == "global":
+                            resource_counts[result["service"]] = len(result["resources"])
+                        else:
+                            key = f"{result['service']}_{result['region']}"
+                            resource_counts[key] = len(result["resources"])
+                elif not result.get("expected", False):
+                    with lock:
+                        collection_errors.append(result["error"])
+
+                # Advance progress (thread-safe)
+                with lock:
+                    progress.advance(main_task)
+
+        progress.update(main_task, description=f"[bold green]✓ Successfully collected {len(all_resources)} resources")
+
+    # Log summary of collection errors if any (but not expected ones)
+    if collection_errors:
+        logger.debug(f"\nCollection completed with {len(collection_errors)} service(s) unavailable")
+        logger.debug("Services that failed:")
+        for error in collection_errors:
+            logger.debug(f"  - {error['service']} ({error['region']}): {error['error']}")
+
+    # Apply filters if specified
+    total_before_filter = len(all_resources)
+    filters_applied = None
+
+    if resource_filter:
+        logger.debug(f"Applying filters: {resource_filter.get_filter_summary()}")
+        all_resources = resource_filter.apply(all_resources)
+        filter_stats = resource_filter.get_statistics_summary()
+
+        filters_applied = {
+            "date_filters": {
+                "before_date": resource_filter.before_date.isoformat() if resource_filter.before_date else None,
+                "after_date": resource_filter.after_date.isoformat() if resource_filter.after_date else None,
+            },
+            "tag_filters": resource_filter.required_tags,
+            "statistics": filter_stats,
+        }
+
+        logger.debug(
+            f"Filtering complete: {filter_stats['total_collected']} collected, "
+            f"{filter_stats['final_count']} matched filters"
+        )
+
+    # Calculate service counts
+    service_counts: Dict[str, int] = {}
+    for resource in all_resources:
+        service_counts[resource.resource_type] = service_counts.get(resource.resource_type, 0) + 1
+
+    # Create snapshot
+    snapshot = Snapshot(
+        name=name,
+        created_at=datetime.now(timezone.utc),
+        account_id=account_id,
+        regions=regions,
+        resources=all_resources,
+        metadata={
+            "tool": "aws-inventory-manager",
+            "version": "1.0.0",
+            "collectors_used": [c(session, "us-east-1").service_name for c in collectors_to_use],
+            "collection_errors": collection_errors if collection_errors else None,
+        },
+        is_active=set_active,
+        service_counts=service_counts,
+        filters_applied=filters_applied,
+        total_resources_before_filter=total_before_filter if resource_filter else None,
+        inventory_name=inventory_name,
+    )
+
+    logger.debug(f"Snapshot '{name}' created with {len(all_resources)} resources")
+
+    return snapshot
+
+
+def create_snapshot_mvp(
+    name: str,
+    regions: List[str],
+    account_id: str,
+    profile_name: Optional[str] = None,
+    set_active: bool = True,
+) -> Snapshot:
+    """Create snapshot using the full implementation.
+
+    This is a wrapper for backward compatibility with the MVP CLI code.
+
+    Args:
+        name: Snapshot name
+        regions: List of AWS regions to scan
+        account_id: AWS account ID
+        profile_name: AWS profile name (optional)
+        set_active: Whether to set as active baseline
+
+    Returns:
+        Snapshot instance with captured resources
+    """
+    return create_snapshot(
+        name=name,
+        regions=regions,
+        account_id=account_id,
+        profile_name=profile_name,
+        set_active=set_active,
+    )
+
+
+def _get_collectors(resource_types: Optional[List[str]] = None) -> List[Type[BaseResourceCollector]]:
+    """Get list of collectors to use based on resource type filter.
+
+    Args:
+        resource_types: Optional list of service names to filter (e.g., ['iam', 'lambda'])
+
+    Returns:
+        List of collector classes to use
+    """
+    if not resource_types:
+        return COLLECTOR_REGISTRY
+
+    # Filter collectors based on service name
+    filtered = []
+    for collector_class in COLLECTOR_REGISTRY:
+        # Create temporary instance to check service name
+        temp_collector = collector_class(boto3.Session(), "us-east-1")
+        if temp_collector.service_name in resource_types:
+            filtered.append(collector_class)
+
+    return filtered if filtered else COLLECTOR_REGISTRY