aws-inventory-manager 0.17.12__py3-none-any.whl

src/utils/unsupported_resources.py

@@ -0,0 +1,306 @@
+"""Utility for detecting unsupported AWS resources.
+
+This module helps identify AWS resources that exist in an account but are not
+covered by any of our collectors. This is important for:
+- Ensuring complete inventory coverage
+- Identifying when new AWS services need collectors
+- Alerting users about resources that won't be included in snapshots
+"""
+
+import logging
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple
+
+import boto3
+from botocore.exceptions import ClientError
+
+logger = logging.getLogger(__name__)
+
+
+# Mapping of AWS resource type prefixes to our collector service names
+# This maps from AWS Resource Groups Tagging API format to our collectors
+SUPPORTED_RESOURCE_TYPE_PREFIXES: Dict[str, str] = {
+    # IAM
+    "iam:": "iam",
+    # Compute
+    "ec2:": "ec2",
+    "lambda:": "lambda",
+    "ecs:": "ecs",
+    "eks:": "eks",
+    # Storage
+    "s3:": "s3",
+    "dynamodb:": "dynamodb",
+    "elasticache:": "elasticache",
+    "rds:": "rds",
+    "elasticfilesystem:": "efs",
+    "backup:": "backup",
+    # Networking
+    "elasticloadbalancing:": "elb",
+    "route53:": "route53",
+    # Messaging
+    "sns:": "sns",
+    "sqs:": "sqs",
+    # Security
+    "secretsmanager:": "secretsmanager",
+    "kms:": "kms",
+    "wafv2:": "waf",
+    # Monitoring & Logging
+    "cloudwatch:": "cloudwatch",
+    "logs:": "cloudwatch",  # CloudWatch Logs is part of CloudWatch collector
+    # Integration & Orchestration
+    "states:": "stepfunctions",
+    "events:": "eventbridge",
+    "apigateway:": "apigateway",
+    "codepipeline:": "codepipeline",
+    "codebuild:": "codebuild",
+    # Management
+    "cloudformation:": "cloudformation",
+    "ssm:": "ssm",
+    # VPC
+    "ec2:vpc-endpoint": "vpcendpoints",
+}
+
+
+@dataclass
+class UnsupportedResource:
+    """Represents an AWS resource that is not covered by any collector."""
+
+    resource_arn: str
+    resource_type: str
+    tags: Dict[str, str]
+    region: str
+
+
+@dataclass
+class UnsupportedResourceReport:
+    """Report of unsupported resources found in an AWS account."""
+
+    unsupported_resources: List[UnsupportedResource]
+    unsupported_types: Set[str]
+    supported_types: Set[str]
+    total_resources_scanned: int
+
+
+def get_all_tagged_resources(
+    session: Optional[boto3.Session] = None,
+    regions: Optional[List[str]] = None,
+) -> List[Tuple[str, str, Dict[str, str], str]]:
+    """Get all tagged resources across all services using Resource Groups Tagging API.
+
+    Args:
+        session: Optional boto3 session (uses default if not provided)
+        regions: Optional list of regions to scan (uses all regions if not provided)
+
+    Returns:
+        List of tuples (arn, resource_type, tags, region)
+    """
+    if session is None:
+        session = boto3.Session()
+
+    if regions is None:
+        # Get all available regions
+        ec2 = session.client("ec2", region_name="us-east-1")
+        try:
+            response = ec2.describe_regions(AllRegions=False)
+            regions = [r["RegionName"] for r in response["Regions"]]
+        except ClientError as e:
+            logger.warning(f"Could not get regions, using default list: {e}")
+            regions = ["us-east-1", "us-west-2", "eu-west-1"]
+
+    all_resources: List[Tuple[str, str, Dict[str, str], str]] = []
+
+    for region in regions:
+        try:
+            tagging = session.client("resourcegroupstaggingapi", region_name=region)
+            paginator = tagging.get_paginator("get_resources")
+
+            for page in paginator.paginate():
+                for resource in page.get("ResourceTagMappingList", []):
+                    arn = resource["ResourceARN"]
+                    tags = {t["Key"]: t["Value"] for t in resource.get("Tags", [])}
+
+                    # Extract resource type from ARN
+                    # ARN format: arn:partition:service:region:account:resource-type/resource-id
+                    parts = arn.split(":")
+                    if len(parts) >= 6:
+                        service = parts[2]
+                        resource_part = parts[5] if len(parts) > 5 else ""
+
+                        # Handle resource types like "bucket/name" or "function:name"
+                        if "/" in resource_part:
+                            resource_type = f"{service}:{resource_part.split('/')[0]}"
+                        elif ":" in resource_part:
+                            resource_type = f"{service}:{resource_part.split(':')[0]}"
+                        else:
+                            resource_type = f"{service}:{resource_part}"
+
+                        all_resources.append((arn, resource_type, tags, region))
+
+        except ClientError as e:
+            logger.debug(f"Could not scan region {region}: {e}")
+        except Exception as e:
+            logger.warning(f"Error scanning region {region}: {e}")
+
+    return all_resources
+
+
+def is_resource_type_supported(resource_type: str) -> bool:
+    """Check if a resource type is supported by any collector.
+
+    Args:
+        resource_type: The AWS resource type (e.g., "s3:bucket", "lambda:function")
+
+    Returns:
+        True if the resource type is supported
+    """
+    resource_type_lower = resource_type.lower()
+
+    for prefix in SUPPORTED_RESOURCE_TYPE_PREFIXES:
+        if resource_type_lower.startswith(prefix):
+            return True
+
+    return False
+
+
+def detect_unsupported_resources(
+    session: Optional[boto3.Session] = None,
+    regions: Optional[List[str]] = None,
+    include_untagged_warning: bool = True,
+) -> UnsupportedResourceReport:
+    """Detect AWS resources that are not supported by any collector.
+
+    This function uses the Resource Groups Tagging API to find all tagged resources
+    and compares them against our supported resource types.
+
+    Note: This only detects TAGGED resources. Resources without tags will not be
+    detected by this method. Use AWS Config for more comprehensive detection.
+
+    Args:
+        session: Optional boto3 session
+        regions: Optional list of regions to scan
+        include_untagged_warning: Whether to log a warning about untagged resources
+
+    Returns:
+        UnsupportedResourceReport with details about unsupported resources
+    """
+    if include_untagged_warning:
+        logger.info(
+            "Note: This detection only covers TAGGED resources. "
+            "Untagged resources will not be detected. "
+            "Consider enabling AWS Config for comprehensive resource tracking."
+        )
+
+    # Get all tagged resources
+    all_resources = get_all_tagged_resources(session, regions)
+
+    unsupported_resources: List[UnsupportedResource] = []
+    unsupported_types: Set[str] = set()
+    supported_types: Set[str] = set()
+
+    for arn, resource_type, tags, region in all_resources:
+        if is_resource_type_supported(resource_type):
+            supported_types.add(resource_type)
+        else:
+            unsupported_types.add(resource_type)
+            unsupported_resources.append(
+                UnsupportedResource(
+                    resource_arn=arn,
+                    resource_type=resource_type,
+                    tags=tags,
+                    region=region,
+                )
+            )
+
+    return UnsupportedResourceReport(
+        unsupported_resources=unsupported_resources,
+        unsupported_types=unsupported_types,
+        supported_types=supported_types,
+        total_resources_scanned=len(all_resources),
+    )
+
+
+def get_unsupported_resource_summary(report: UnsupportedResourceReport) -> str:
+    """Generate a human-readable summary of unsupported resources.
+
+    Args:
+        report: The unsupported resource report
+
+    Returns:
+        Formatted string summary
+    """
+    lines = [
+        f"Unsupported Resource Detection Report",
+        "=" * 40,
+        f"Total resources scanned: {report.total_resources_scanned}",
+        f"Supported resource types found: {len(report.supported_types)}",
+        f"Unsupported resource types found: {len(report.unsupported_types)}",
+        f"Total unsupported resources: {len(report.unsupported_resources)}",
+        "",
+    ]
+
+    if report.unsupported_types:
+        lines.append("Unsupported Resource Types:")
+        lines.append("-" * 30)
+        for resource_type in sorted(report.unsupported_types):
+            count = sum(
+                1
+                for r in report.unsupported_resources
+                if r.resource_type == resource_type
+            )
+            lines.append(f"  {resource_type}: {count} resource(s)")
+
+        lines.append("")
+        lines.append("Consider adding collectors for these services to ensure")
+        lines.append("complete inventory coverage and safe cleanup operations.")
+
+    return "\n".join(lines)
+
+
+def check_for_unsupported_resources_quick(
+    session: Optional[boto3.Session] = None,
+    region: str = "us-east-1",
+    limit: int = 100,
+) -> Tuple[bool, Set[str]]:
+    """Quick check for unsupported resources in a single region.
+
+    This is a faster alternative to full detection, useful for CLI warnings.
+
+    Args:
+        session: Optional boto3 session
+        region: Region to check
+        limit: Maximum number of resources to check
+
+    Returns:
+        Tuple of (has_unsupported, unsupported_types)
+    """
+    if session is None:
+        session = boto3.Session()
+
+    unsupported_types: Set[str] = set()
+
+    try:
+        tagging = session.client("resourcegroupstaggingapi", region_name=region)
+
+        response = tagging.get_resources(ResourcesPerPage=limit)
+
+        for resource in response.get("ResourceTagMappingList", []):
+            arn = resource["ResourceARN"]
+            parts = arn.split(":")
+            if len(parts) >= 6:
+                service = parts[2]
+                resource_part = parts[5] if len(parts) > 5 else ""
+
+                if "/" in resource_part:
+                    resource_type = f"{service}:{resource_part.split('/')[0]}"
+                elif ":" in resource_part:
+                    resource_type = f"{service}:{resource_part.split(':')[0]}"
+                else:
+                    resource_type = f"{service}:{resource_part}"
+
+                if not is_resource_type_supported(resource_type):
+                    unsupported_types.add(resource_type)
+
+    except Exception as e:
+        logger.debug(f"Quick unsupported resource check failed: {e}")
+
+    return bool(unsupported_types), unsupported_types

src/web/__init__.py

@@ -0,0 +1,5 @@
+"""AWS Inventory Browser - Web UI module."""
+
+from .app import create_app
+
+__all__ = ["create_app"]

src/web/app.py

@@ -0,0 +1,97 @@
+"""FastAPI application factory for AWS Inventory Browser."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Optional
+
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+
+from .dependencies import init_database
+
+# Get the directory where this module is located
+MODULE_DIR = Path(__file__).parent
+TEMPLATES_DIR = MODULE_DIR / "templates"
+STATIC_DIR = MODULE_DIR / "static"
+
+
+def get_templates() -> Jinja2Templates:
+    """Get configured Jinja2Templates instance."""
+    templates = Jinja2Templates(directory=str(TEMPLATES_DIR))
+
+    # Add custom filters
+    def format_number(value: int) -> str:
+        """Format number with commas."""
+        return f"{value:,}"
+
+    def truncate_arn(arn: str, max_length: int = 50) -> str:
+        """Truncate ARN for display."""
+        if len(arn) <= max_length:
+            return arn
+        return arn[:max_length - 3] + "..."
+
+    def service_from_type(resource_type: str) -> str:
+        """Extract service name from resource type."""
+        if ":" in resource_type:
+            return resource_type.split(":")[0]
+        return resource_type
+
+    templates.env.filters["format_number"] = format_number
+    templates.env.filters["truncate_arn"] = truncate_arn
+    templates.env.filters["service"] = service_from_type
+
+    return templates
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage application lifecycle."""
+    # Startup: Initialize database
+    storage_path = getattr(app.state, "storage_path", None)
+    init_database(storage_path)
+    yield
+    # Shutdown: Nothing to clean up
+
+
+def create_app(storage_path: Optional[str] = None) -> FastAPI:
+    """Create and configure the FastAPI application.
+
+    Args:
+        storage_path: Optional path to storage directory.
+                     If not provided, uses default from Config.
+
+    Returns:
+        Configured FastAPI application instance.
+    """
+    app = FastAPI(
+        title="AWS Inventory Browser",
+        description="Browse and analyze AWS resource inventory",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    # Store config for lifespan access
+    app.state.storage_path = storage_path
+
+    # Mount static files if directory exists
+    if STATIC_DIR.exists():
+        app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
+
+    # Get templates
+    templates = get_templates()
+
+    # Include API routes
+    from .routes.api import router as api_router
+    app.include_router(api_router, prefix="/api")
+
+    # Include page routes
+    from .routes import pages
+    app.include_router(pages.router)
+
+    # Add templates to app state for access in routes
+    app.state.templates = templates
+
+    return app

src/web/dependencies.py

@@ -0,0 +1,69 @@
+"""Dependency injection for FastAPI routes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Optional
+
+from ..storage import AuditStore, Database, GroupStore, InventoryStore, ResourceStore, SnapshotStore
+
+# Global instances (initialized at startup)
+_db: Optional[Database] = None
+_storage_path: Optional[str] = None
+
+
+def init_database(storage_path: Optional[str] = None) -> None:
+    """Initialize the database connection.
+
+    Args:
+        storage_path: Optional path to storage directory.
+                     If not provided, uses default from Config.
+    """
+    global _db, _storage_path
+    _storage_path = storage_path
+
+    if storage_path:
+        db_path = Path(storage_path) / "inventory.db"
+        _db = Database(db_path=db_path)
+    else:
+        _db = Database()
+
+    _db.ensure_schema()
+
+
+def get_database() -> Database:
+    """Get the database instance."""
+    global _db
+    if _db is None:
+        init_database(_storage_path)
+    return _db  # type: ignore
+
+
+def get_snapshot_store() -> SnapshotStore:
+    """Get a SnapshotStore instance."""
+    return SnapshotStore(get_database())
+
+
+def get_resource_store() -> ResourceStore:
+    """Get a ResourceStore instance."""
+    return ResourceStore(get_database())
+
+
+def get_inventory_store() -> InventoryStore:
+    """Get an InventoryStore instance."""
+    return InventoryStore(get_database())
+
+
+def get_audit_store() -> AuditStore:
+    """Get an AuditStore instance."""
+    return AuditStore(get_database())
+
+
+def get_group_store() -> GroupStore:
+    """Get a GroupStore instance."""
+    return GroupStore(get_database())
+
+
+def get_storage_path() -> Optional[str]:
+    """Get the configured storage path."""
+    return _storage_path

src/web/routes/__init__.py

@@ -0,0 +1 @@
+"""Web routes package."""

src/web/routes/api/__init__.py

@@ -0,0 +1,18 @@
+"""API routes package."""
+
+from fastapi import APIRouter
+
+from . import charts, cleanup, filters, groups, inventories, queries, resources, snapshots, views
+
+router = APIRouter()
+router.include_router(snapshots.router, tags=["snapshots"])
+router.include_router(resources.router, tags=["resources"])
+router.include_router(queries.router, tags=["queries"])
+router.include_router(charts.router, tags=["charts"])
+router.include_router(cleanup.router, tags=["cleanup"])
+router.include_router(filters.router, tags=["filters"])
+router.include_router(views.router, tags=["views"])
+router.include_router(groups.router, tags=["groups"])
+router.include_router(inventories.router, tags=["inventories"])
+
+__all__ = ["router"]

src/web/routes/api/charts.py

@@ -0,0 +1,156 @@
+"""Chart data API endpoints."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from fastapi import APIRouter, Query
+
+from ...dependencies import get_resource_store, get_snapshot_store
+
+router = APIRouter(prefix="/charts")
+
+# Chart.js color palette
+CHART_COLORS = [
+    "#3B82F6",  # Blue
+    "#10B981",  # Green
+    "#F59E0B",  # Amber
+    "#EF4444",  # Red
+    "#8B5CF6",  # Purple
+    "#EC4899",  # Pink
+    "#06B6D4",  # Cyan
+    "#84CC16",  # Lime
+    "#F97316",  # Orange
+    "#6366F1",  # Indigo
+]
+
+
+@router.get("/resources-by-type")
+async def chart_resources_by_type(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    limit: int = Query(10, le=20),
+):
+    """Get Chart.js data for resources by type."""
+    store = get_resource_store()
+    stats = store.get_stats(snapshot_name=snapshot, group_by="type")[:limit]
+
+    labels = [s["group_key"] for s in stats]
+    data = [s["count"] for s in stats]
+
+    return {
+        "labels": labels,
+        "datasets": [
+            {
+                "data": data,
+                "backgroundColor": CHART_COLORS[:len(data)],
+            }
+        ],
+    }
+
+
+@router.get("/resources-by-region")
+async def chart_resources_by_region(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get Chart.js data for resources by region."""
+    store = get_resource_store()
+    stats = store.get_stats(snapshot_name=snapshot, group_by="region")
+
+    labels = [s["group_key"] for s in stats]
+    data = [s["count"] for s in stats]
+
+    return {
+        "labels": labels,
+        "datasets": [
+            {
+                "label": "Resources",
+                "data": data,
+                "backgroundColor": "#3B82F6",
+            }
+        ],
+    }
+
+
+@router.get("/resources-by-service")
+async def chart_resources_by_service(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    limit: int = Query(10, le=20),
+):
+    """Get Chart.js data for resources by service."""
+    store = get_resource_store()
+    stats = store.get_stats(snapshot_name=snapshot, group_by="service")[:limit]
+
+    labels = [s["group_key"] for s in stats]
+    data = [s["count"] for s in stats]
+
+    return {
+        "labels": labels,
+        "datasets": [
+            {
+                "data": data,
+                "backgroundColor": CHART_COLORS[:len(data)],
+            }
+        ],
+    }
+
+
+@router.get("/snapshot-trend")
+async def chart_snapshot_trend():
+    """Get Chart.js data for snapshot resource count over time."""
+    store = get_snapshot_store()
+    snapshots = store.list_all()
+
+    # Sort by created_at
+    sorted_snapshots = sorted(snapshots, key=lambda s: s.get("created_at", ""))
+
+    # Take last 10 snapshots for trend
+    recent = sorted_snapshots[-10:] if len(sorted_snapshots) > 10 else sorted_snapshots
+
+    labels = [s["name"][:20] for s in recent]  # Truncate long names
+    data = [s.get("resource_count", 0) for s in recent]
+
+    return {
+        "labels": labels,
+        "datasets": [
+            {
+                "label": "Resource Count",
+                "data": data,
+                "borderColor": "#3B82F6",
+                "backgroundColor": "rgba(59, 130, 246, 0.1)",
+                "fill": True,
+                "tension": 0.3,
+            }
+        ],
+    }
+
+
+@router.get("/tag-coverage")
+async def chart_tag_coverage(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get Chart.js data for tag coverage analysis."""
+    resource_store = get_resource_store()
+
+    # Get all resources
+    resources = resource_store.search(snapshot_name=snapshot, limit=10000)
+
+    # Count tagged vs untagged
+    tagged = 0
+    untagged = 0
+
+    for r in resources:
+        tags = r.get("tags", {})
+        if tags and len(tags) > 0:
+            tagged += 1
+        else:
+            untagged += 1
+
+    return {
+        "labels": ["Tagged", "Untagged"],
+        "datasets": [
+            {
+                "data": [tagged, untagged],
+                "backgroundColor": ["#10B981", "#EF4444"],
+            }
+        ],
+    }