aws-inventory-manager 0.13.2__py3-none-any.whl

src/utils/hash.py

@@ -0,0 +1,60 @@
+"""Configuration hashing utility for change detection."""
+
+import hashlib
+import json
+from typing import Any, Dict, Set
+
+# Attributes to exclude from hashing (volatile data)
+EXCLUDE_ATTRIBUTES: Set[str] = {
+    "ResponseMetadata",
+    "LastModifiedDate",
+    "CreatedDate",
+    "CreateDate",
+    "State",
+    "Status",
+    "RequestId",
+    "VersionId",
+    "LastUpdateTime",
+    "LastUpdatedTime",
+    "ModifiedTime",
+}
+
+
+def compute_config_hash(resource_data: Dict[str, Any]) -> str:
+    """Compute stable SHA256 hash of resource configuration.
+
+    This hash is used for change detection. Volatile attributes
+    (timestamps, states, etc.) are excluded to prevent false positives.
+
+    Args:
+        resource_data: Resource configuration dictionary
+
+    Returns:
+        64-character SHA256 hex string
+    """
+    # Deep copy and remove excluded attributes
+    clean_data = _remove_volatile_attributes(resource_data, EXCLUDE_ATTRIBUTES)
+
+    # Normalize: sort keys for deterministic JSON
+    normalized = json.dumps(clean_data, sort_keys=True, default=str)
+
+    # Hash
+    return hashlib.sha256(normalized.encode()).hexdigest()
+
+
+def _remove_volatile_attributes(data: Any, exclude_set: Set[str]) -> Any:
+    """Recursively remove excluded attributes from nested dict/list.
+
+    Args:
+        data: Data structure to clean (dict, list, or primitive)
+        exclude_set: Set of attribute names to exclude
+
+    Returns:
+        Cleaned data structure
+    """
+    if isinstance(data, dict):
+        return {k: _remove_volatile_attributes(v, exclude_set) for k, v in data.items() if k not in exclude_set}
+    elif isinstance(data, list):
+        return [_remove_volatile_attributes(item, exclude_set) for item in data]
+    else:
+        return data

src/utils/logging.py

@@ -0,0 +1,63 @@
+"""Logging configuration for AWS Baseline Snapshot tool."""
+
+import logging
+import sys
+from typing import Optional
+
+
+def setup_logging(level: str = "INFO", log_file: Optional[str] = None, verbose: bool = False) -> None:
+    """Configure logging for the application.
+
+    Args:
+        level: Log level (DEBUG, INFO, WARN, ERROR)
+        log_file: Optional log file path
+        verbose: If True, show detailed logs; if False, suppress all but critical
+    """
+    # Convert string level to logging constant
+    numeric_level = getattr(logging, level.upper(), logging.INFO)
+
+    # In non-verbose mode, suppress all logs except CRITICAL
+    # User will only see styled Rich console output
+    if not verbose:
+        numeric_level = logging.CRITICAL
+
+    # Create formatter
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")
+
+    # Configure root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(numeric_level)
+
+    # Remove existing handlers
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Console handler (only add if verbose or log_file specified)
+    if verbose or log_file:
+        console_handler = logging.StreamHandler(sys.stderr)
+        console_handler.setLevel(numeric_level)
+        console_handler.setFormatter(formatter)
+        root_logger.addHandler(console_handler)
+
+    # File handler (if specified)
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setLevel(logging.DEBUG)  # Always log everything to file
+        file_handler.setFormatter(formatter)
+        root_logger.addHandler(file_handler)
+
+    # Suppress noisy third-party loggers
+    logging.getLogger("boto3").setLevel(logging.CRITICAL)
+    logging.getLogger("botocore").setLevel(logging.CRITICAL)
+    logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+    logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
+
+    # Suppress internal module logs unless verbose
+    if not verbose:
+        logging.getLogger("src").setLevel(logging.CRITICAL)
+        logging.getLogger("src.snapshot").setLevel(logging.CRITICAL)
+        logging.getLogger("src.snapshot.resource_collectors").setLevel(logging.CRITICAL)
+        logging.getLogger("src.snapshot.capturer").setLevel(logging.CRITICAL)
+        logging.getLogger("src.snapshot.storage").setLevel(logging.CRITICAL)
+        logging.getLogger("src.aws").setLevel(logging.CRITICAL)
+        logging.getLogger("src.aws.credentials").setLevel(logging.CRITICAL)

src/utils/pagination.py

@@ -0,0 +1,41 @@
+"""
+Terminal pagination utilities for large resource lists.
+
+This module provides pagination functionality for displaying large datasets
+in the terminal with user-friendly navigation controls.
+"""
+
+from __future__ import annotations
+
+from typing import Generator, List, TypeVar
+
+T = TypeVar("T")
+
+
+def paginate_resources(items: List[T], page_size: int = 100) -> Generator[List[T], None, None]:
+    """
+    Paginate a list of items into pages of specified size.
+
+    This is a memory-efficient generator that yields pages of items
+    without loading everything into memory at once.
+
+    Args:
+        items: List of items to paginate
+        page_size: Number of items per page (default: 100)
+
+    Yields:
+        Lists of items, each containing up to page_size items
+
+    Example:
+        >>> resources = list(range(250))
+        >>> for page in paginate_resources(resources, page_size=100):
+        ...     print(f"Page has {len(page)} items")
+        Page has 100 items
+        Page has 100 items
+        Page has 50 items
+    """
+    if not items:
+        return
+
+    for i in range(0, len(items), page_size):
+        yield items[i : i + page_size]

src/utils/paths.py

@@ -0,0 +1,51 @@
+"""Path resolution utilities for snapshot storage."""
+
+import os
+from pathlib import Path
+from typing import Optional, Union
+
+
+def get_snapshot_storage_path(custom_path: Optional[Union[str, Path]] = None) -> Path:
+    """Resolve snapshot storage path with precedence: parameter > env var > default.
+
+    Precedence order:
+    1. custom_path parameter (if provided)
+    2. AWS_INVENTORY_STORAGE_PATH environment variable (if set)
+    3. ~/.snapshots (default)
+
+    Args:
+        custom_path: Optional custom path override
+
+    Returns:
+        Resolved Path object for snapshot storage
+
+    Examples:
+        # Use default
+        >>> get_snapshot_storage_path()
+        Path.home() / '.snapshots'
+
+        # Use environment variable
+        >>> os.environ['AWS_INVENTORY_STORAGE_PATH'] = '/data/snapshots'
+        >>> get_snapshot_storage_path()
+        Path('/data/snapshots')
+
+        # Use parameter (highest priority)
+        >>> get_snapshot_storage_path('/custom/path')
+        Path('/custom/path')
+    """
+    # Priority 1: Custom path parameter (but not empty string)
+    if custom_path:
+        # Handle both str and Path types
+        if isinstance(custom_path, str):
+            if custom_path.strip():
+                return Path(custom_path).expanduser().resolve()
+        else:  # Path object
+            return custom_path.expanduser().resolve()
+
+    # Priority 2: Environment variable
+    env_path = os.getenv("AWS_INVENTORY_STORAGE_PATH")
+    if env_path:
+        return Path(env_path).expanduser().resolve()
+
+    # Priority 3: Default to ~/.snapshots
+    return Path.home() / ".snapshots"

src/utils/progress.py

@@ -0,0 +1,41 @@
+"""Progress indicator utilities using Rich library."""
+
+from contextlib import contextmanager
+
+from rich.progress import (
+    BarColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeRemainingColumn,
+)
+
+
+@contextmanager
+def create_progress():
+    """Create a Rich progress context for tracking operations.
+
+    Yields:
+        Progress instance configured for multi-task tracking
+    """
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeRemainingColumn(),
+    ) as progress:
+        yield progress
+
+
+def create_spinner_progress():
+    """Create a simple spinner progress for indeterminate operations.
+
+    Returns:
+        Progress instance with spinner
+    """
+    return Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+    )

src/utils/unsupported_resources.py

@@ -0,0 +1,306 @@
+"""Utility for detecting unsupported AWS resources.
+
+This module helps identify AWS resources that exist in an account but are not
+covered by any of our collectors. This is important for:
+- Ensuring complete inventory coverage
+- Identifying when new AWS services need collectors
+- Alerting users about resources that won't be included in snapshots
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple
+
+import boto3
+from botocore.exceptions import ClientError
+
+logger = logging.getLogger(__name__)
+
+
+# Mapping of AWS resource type prefixes to our collector service names
+# This maps from AWS Resource Groups Tagging API format to our collectors
+SUPPORTED_RESOURCE_TYPE_PREFIXES: Dict[str, str] = {
+    # IAM
+    "iam:": "iam",
+    # Compute
+    "ec2:": "ec2",
+    "lambda:": "lambda",
+    "ecs:": "ecs",
+    "eks:": "eks",
+    # Storage
+    "s3:": "s3",
+    "dynamodb:": "dynamodb",
+    "elasticache:": "elasticache",
+    "rds:": "rds",
+    "elasticfilesystem:": "efs",
+    "backup:": "backup",
+    # Networking
+    "elasticloadbalancing:": "elb",
+    "route53:": "route53",
+    # Messaging
+    "sns:": "sns",
+    "sqs:": "sqs",
+    # Security
+    "secretsmanager:": "secretsmanager",
+    "kms:": "kms",
+    "wafv2:": "waf",
+    # Monitoring & Logging
+    "cloudwatch:": "cloudwatch",
+    "logs:": "cloudwatch",  # CloudWatch Logs is part of CloudWatch collector
+    # Integration & Orchestration
+    "states:": "stepfunctions",
+    "events:": "eventbridge",
+    "apigateway:": "apigateway",
+    "codepipeline:": "codepipeline",
+    "codebuild:": "codebuild",
+    # Management
+    "cloudformation:": "cloudformation",
+    "ssm:": "ssm",
+    # VPC
+    "ec2:vpc-endpoint": "vpcendpoints",
+}
+
+
+@dataclass
+class UnsupportedResource:
+    """Represents an AWS resource that is not covered by any collector."""
+
+    resource_arn: str
+    resource_type: str
+    tags: Dict[str, str]
+    region: str
+
+
+@dataclass
+class UnsupportedResourceReport:
+    """Report of unsupported resources found in an AWS account."""
+
+    unsupported_resources: List[UnsupportedResource]
+    unsupported_types: Set[str]
+    supported_types: Set[str]
+    total_resources_scanned: int
+
+
+def get_all_tagged_resources(
+    session: Optional[boto3.Session] = None,
+    regions: Optional[List[str]] = None,
+) -> List[Tuple[str, str, Dict[str, str], str]]:
+    """Get all tagged resources across all services using Resource Groups Tagging API.
+
+    Args:
+        session: Optional boto3 session (uses default if not provided)
+        regions: Optional list of regions to scan (uses all regions if not provided)
+
+    Returns:
+        List of tuples (arn, resource_type, tags, region)
+    """
+    if session is None:
+        session = boto3.Session()
+
+    if regions is None:
+        # Get all available regions
+        ec2 = session.client("ec2", region_name="us-east-1")
+        try:
+            response = ec2.describe_regions(AllRegions=False)
+            regions = [r["RegionName"] for r in response["Regions"]]
+        except ClientError as e:
+            logger.warning(f"Could not get regions, using default list: {e}")
+            regions = ["us-east-1", "us-west-2", "eu-west-1"]
+
+    all_resources: List[Tuple[str, str, Dict[str, str], str]] = []
+
+    for region in regions:
+        try:
+            tagging = session.client("resourcegroupstaggingapi", region_name=region)
+            paginator = tagging.get_paginator("get_resources")
+
+            for page in paginator.paginate():
+                for resource in page.get("ResourceTagMappingList", []):
+                    arn = resource["ResourceARN"]
+                    tags = {t["Key"]: t["Value"] for t in resource.get("Tags", [])}
+
+                    # Extract resource type from ARN
+                    # ARN format: arn:partition:service:region:account:resource-type/resource-id
+                    parts = arn.split(":")
+                    if len(parts) >= 6:
+                        service = parts[2]
+                        resource_part = parts[5] if len(parts) > 5 else ""
+
+                        # Handle resource types like "bucket/name" or "function:name"
+                        if "/" in resource_part:
+                            resource_type = f"{service}:{resource_part.split('/')[0]}"
+                        elif ":" in resource_part:
+                            resource_type = f"{service}:{resource_part.split(':')[0]}"
+                        else:
+                            resource_type = f"{service}:{resource_part}"
+
+                        all_resources.append((arn, resource_type, tags, region))
+
+        except ClientError as e:
+            logger.debug(f"Could not scan region {region}: {e}")
+        except Exception as e:
+            logger.warning(f"Error scanning region {region}: {e}")
+
+    return all_resources
+
+
+def is_resource_type_supported(resource_type: str) -> bool:
+    """Check if a resource type is supported by any collector.
+
+    Args:
+        resource_type: The AWS resource type (e.g., "s3:bucket", "lambda:function")
+
+    Returns:
+        True if the resource type is supported
+    """
+    resource_type_lower = resource_type.lower()
+
+    for prefix in SUPPORTED_RESOURCE_TYPE_PREFIXES:
+        if resource_type_lower.startswith(prefix):
+            return True
+
+    return False
+
+
+def detect_unsupported_resources(
+    session: Optional[boto3.Session] = None,
+    regions: Optional[List[str]] = None,
+    include_untagged_warning: bool = True,
+) -> UnsupportedResourceReport:
+    """Detect AWS resources that are not supported by any collector.
+
+    This function uses the Resource Groups Tagging API to find all tagged resources
+    and compares them against our supported resource types.
+
+    Note: This only detects TAGGED resources. Resources without tags will not be
+    detected by this method. Use AWS Config for more comprehensive detection.
+
+    Args:
+        session: Optional boto3 session
+        regions: Optional list of regions to scan
+        include_untagged_warning: Whether to log a warning about untagged resources
+
+    Returns:
+        UnsupportedResourceReport with details about unsupported resources
+    """
+    if include_untagged_warning:
+        logger.info(
+            "Note: This detection only covers TAGGED resources. "
+            "Untagged resources will not be detected. "
+            "Consider enabling AWS Config for comprehensive resource tracking."
+        )
+
+    # Get all tagged resources
+    all_resources = get_all_tagged_resources(session, regions)
+
+    unsupported_resources: List[UnsupportedResource] = []
+    unsupported_types: Set[str] = set()
+    supported_types: Set[str] = set()
+
+    for arn, resource_type, tags, region in all_resources:
+        if is_resource_type_supported(resource_type):
+            supported_types.add(resource_type)
+        else:
+            unsupported_types.add(resource_type)
+            unsupported_resources.append(
+                UnsupportedResource(
+                    resource_arn=arn,
+                    resource_type=resource_type,
+                    tags=tags,
+                    region=region,
+                )
+            )
+
+    return UnsupportedResourceReport(
+        unsupported_resources=unsupported_resources,
+        unsupported_types=unsupported_types,
+        supported_types=supported_types,
+        total_resources_scanned=len(all_resources),
+    )
+
+
+def get_unsupported_resource_summary(report: UnsupportedResourceReport) -> str:
+    """Generate a human-readable summary of unsupported resources.
+
+    Args:
+        report: The unsupported resource report
+
+    Returns:
+        Formatted string summary
+    """
+    lines = [
+        f"Unsupported Resource Detection Report",
+        "=" * 40,
+        f"Total resources scanned: {report.total_resources_scanned}",
+        f"Supported resource types found: {len(report.supported_types)}",
+        f"Unsupported resource types found: {len(report.unsupported_types)}",
+        f"Total unsupported resources: {len(report.unsupported_resources)}",
+        "",
+    ]
+
+    if report.unsupported_types:
+        lines.append("Unsupported Resource Types:")
+        lines.append("-" * 30)
+        for resource_type in sorted(report.unsupported_types):
+            count = sum(
+                1
+                for r in report.unsupported_resources
+                if r.resource_type == resource_type
+            )
+            lines.append(f"  {resource_type}: {count} resource(s)")
+
+        lines.append("")
+        lines.append("Consider adding collectors for these services to ensure")
+        lines.append("complete inventory coverage and safe cleanup operations.")
+
+    return "\n".join(lines)
+
+
+def check_for_unsupported_resources_quick(
+    session: Optional[boto3.Session] = None,
+    region: str = "us-east-1",
+    limit: int = 100,
+) -> Tuple[bool, Set[str]]:
+    """Quick check for unsupported resources in a single region.
+
+    This is a faster alternative to full detection, useful for CLI warnings.
+
+    Args:
+        session: Optional boto3 session
+        region: Region to check
+        limit: Maximum number of resources to check
+
+    Returns:
+        Tuple of (has_unsupported, unsupported_types)
+    """
+    if session is None:
+        session = boto3.Session()
+
+    unsupported_types: Set[str] = set()
+
+    try:
+        tagging = session.client("resourcegroupstaggingapi", region_name=region)
+
+        response = tagging.get_resources(ResourcesPerPage=limit)
+
+        for resource in response.get("ResourceTagMappingList", []):
+            arn = resource["ResourceARN"]
+            parts = arn.split(":")
+            if len(parts) >= 6:
+                service = parts[2]
+                resource_part = parts[5] if len(parts) > 5 else ""
+
+                if "/" in resource_part:
+                    resource_type = f"{service}:{resource_part.split('/')[0]}"
+                elif ":" in resource_part:
+                    resource_type = f"{service}:{resource_part.split(':')[0]}"
+                else:
+                    resource_type = f"{service}:{resource_part}"
+
+                if not is_resource_type_supported(resource_type):
+                    unsupported_types.add(resource_type)
+
+    except Exception as e:
+        logger.debug(f"Quick unsupported resource check failed: {e}")
+
+    return bool(unsupported_types), unsupported_types

src/web/__init__.py

@@ -0,0 +1,5 @@
+"""AWS Inventory Browser - Web UI module."""
+
+from .app import create_app
+
+__all__ = ["create_app"]

src/web/app.py

@@ -0,0 +1,97 @@
+"""FastAPI application factory for AWS Inventory Browser."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from .dependencies import init_database
+
+# Get the directory where this module is located
+MODULE_DIR = Path(__file__).parent
+TEMPLATES_DIR = MODULE_DIR / "templates"
+STATIC_DIR = MODULE_DIR / "static"
+
+
+def get_templates() -> Jinja2Templates:
+    """Get configured Jinja2Templates instance."""
+    templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
+
+    # Add custom filters
+    def format_number(value: int) -> str:
+        """Format number with commas."""
+        return f"{value:,}"
+
+    def truncate_arn(arn: str, max_length: int = 50) -> str:
+        """Truncate ARN for display."""
+        if len(arn) <= max_length:
+            return arn
+        return arn[:max_length - 3] + "..."
+
+    def service_from_type(resource_type: str) -> str:
+        """Extract service name from resource type."""
+        if ":" in resource_type:
+            return resource_type.split(":")[0]
+        return resource_type
+
+    templates.env.filters["format_number"] = format_number
+    templates.env.filters["truncate_arn"] = truncate_arn
+    templates.env.filters["service"] = service_from_type
+
+    return templates
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage application lifecycle."""
+    # Startup: Initialize database
+    storage_path = getattr(app.state, "storage_path", None)
+    init_database(storage_path)
+    yield
+    # Shutdown: Nothing to clean up
+
+
+def create_app(storage_path: Optional[str] = None) -> FastAPI:
+    """Create and configure the FastAPI application.
+
+    Args:
+        storage_path: Optional path to storage directory.
+                     If not provided, uses default from Config.
+
+    Returns:
+        Configured FastAPI application instance.
+    """
+    app = FastAPI(
+        title="AWS Inventory Browser",
+        description="Browse and analyze AWS resource inventory",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    # Store config for lifespan access
+    app.state.storage_path = storage_path
+
+    # Mount static files if directory exists
+    if STATIC_DIR.exists():
+        app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
+
+    # Get templates
+    templates = get_templates()
+
+    # Include API routes
+    from .routes.api import router as api_router
+    app.include_router(api_router, prefix="/api")
+
+    # Include page routes
+    from .routes import pages
+    app.include_router(pages.router)
+
+    # Add templates to app state for access in routes
+    app.state.templates = templates
+
+    return app