aws-inventory-manager 0.17.12__py3-none-any.whl

src/web/routes/api/resources.py

@@ -0,0 +1,393 @@
+"""Resource API endpoints."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+from urllib.parse import unquote
+
+import yaml
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import StreamingResponse
+
+from ...dependencies import get_database, get_resource_store, get_snapshot_store
+
+router = APIRouter(prefix="/resources")
+
+
+@router.get("")
+async def search_resources(
+    q: Optional[str] = Query(None, description="Search query (ARN pattern)"),
+    type: Optional[str] = Query(None, description="Filter by resource type"),
+    region: Optional[str] = Query(None, description="Filter by region"),
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    tag_key: Optional[str] = Query(None, description="Filter by tag key"),
+    tag_value: Optional[str] = Query(None, description="Filter by tag value"),
+    include_tags: bool = Query(False, description="Include tags in response"),
+    limit: int = Query(50, le=500),
+    offset: int = Query(0, ge=0),
+):
+    """Search resources across snapshots."""
+    store = get_resource_store()
+
+    resources = store.search(
+        arn_pattern=q,
+        resource_type=type,
+        region=region,
+        snapshot_name=snapshot,
+        tag_key=tag_key,
+        tag_value=tag_value,
+        limit=limit,
+        offset=offset,
+    )
+
+    # Optionally include tags for each resource
+    if include_tags:
+        enriched_resources = []
+        for resource in resources:
+            r = dict(resource)
+            tags = store.get_tags_for_resource(
+                resource["arn"],
+                snapshot_name=resource.get("snapshot_name"),
+            )
+            r["tags"] = tags
+            enriched_resources.append(r)
+        resources = enriched_resources
+
+    return {
+        "count": len(resources),
+        "limit": limit,
+        "offset": offset,
+        "resources": resources,
+    }
+
+
+@router.get("/types")
+async def get_resource_types(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get unique resource types."""
+    store = get_resource_store()
+    types = store.get_unique_resource_types(snapshot_name=snapshot)
+    return {"types": types}
+
+
+@router.get("/regions")
+async def get_regions(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get unique regions."""
+    store = get_resource_store()
+    regions = store.get_unique_regions(snapshot_name=snapshot)
+    return {"regions": regions}
+
+
+@router.get("/tags/keys")
+async def get_tag_keys(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get unique tag keys."""
+    db = get_database()
+
+    if snapshot:
+        query = """
+            SELECT DISTINCT t.key
+            FROM resource_tags t
+            JOIN resources r ON t.resource_id = r.id
+            JOIN snapshots s ON r.snapshot_id = s.id
+            WHERE s.name = ?
+            ORDER BY t.key
+        """
+        rows = db.fetchall(query, (snapshot,))
+    else:
+        query = """
+            SELECT DISTINCT key FROM resource_tags ORDER BY key
+        """
+        rows = db.fetchall(query)
+
+    return {"keys": [row["key"] for row in rows]}
+
+
+@router.get("/tags/values")
+async def get_tag_values(
+    key: str = Query(..., description="Tag key to get values for"),
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+):
+    """Get unique values for a tag key."""
+    db = get_database()
+
+    if snapshot:
+        query = """
+            SELECT DISTINCT t.value
+            FROM resource_tags t
+            JOIN resources r ON t.resource_id = r.id
+            JOIN snapshots s ON r.snapshot_id = s.id
+            WHERE t.key = ? AND s.name = ?
+            ORDER BY t.value
+        """
+        rows = db.fetchall(query, (key, snapshot))
+    else:
+        query = """
+            SELECT DISTINCT value FROM resource_tags WHERE key = ? ORDER BY value
+        """
+        rows = db.fetchall(query, (key,))
+
+    return {"key": key, "values": [row["value"] for row in rows]}
+
+
+@router.get("/stats")
+async def get_stats(
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    group_by: str = Query("type", description="Group by: type, region, service"),
+):
+    """Get resource statistics."""
+    store = get_resource_store()
+    stats = store.get_stats(snapshot_name=snapshot, group_by=group_by)
+    return {"group_by": group_by, "stats": stats}
+
+
+@router.get("/by-arn/{arn:path}")
+async def get_resource_by_arn(arn: str):
+    """Get resource details by ARN."""
+    # URL decode the ARN
+    decoded_arn = unquote(arn)
+
+    store = get_resource_store()
+    resources = store.search(arn_pattern=decoded_arn, limit=1)
+
+    if not resources:
+        raise HTTPException(status_code=404, detail=f"Resource not found: {decoded_arn}")
+
+    return resources[0]
+
+
+@router.get("/history/{arn:path}")
+async def get_resource_history(arn: str):
+    """Get resource history across snapshots."""
+    decoded_arn = unquote(arn)
+
+    store = get_resource_store()
+    history = store.get_history(decoded_arn)
+
+    if not history:
+        raise HTTPException(status_code=404, detail=f"No history found for: {decoded_arn}")
+
+    return {"arn": decoded_arn, "snapshots": history}
+
+
+@router.get("/diff")
+async def compare_snapshots(
+    snapshot1: str = Query(..., description="First snapshot name"),
+    snapshot2: str = Query(..., description="Second snapshot name"),
+    type: Optional[str] = Query(None, description="Filter by resource type"),
+):
+    """Compare resources between two snapshots."""
+    snapshot_store = get_snapshot_store()
+
+    # Validate snapshots exist
+    if not snapshot_store.exists(snapshot1):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{snapshot1}' not found")
+    if not snapshot_store.exists(snapshot2):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{snapshot2}' not found")
+
+    resource_store = get_resource_store()
+    diff = resource_store.compare_snapshots(snapshot1, snapshot2)
+
+    # Filter by type if specified
+    if type:
+        diff["added"] = [r for r in diff["added"] if r.get("resource_type") == type]
+        diff["removed"] = [r for r in diff["removed"] if r.get("resource_type") == type]
+        diff["modified"] = [r for r in diff["modified"] if r.get("resource_type") == type]
+
+    return diff
+
+
+@router.get("/export/csv")
+async def export_resources_csv(
+    q: Optional[str] = Query(None, description="Search query (ARN pattern)"),
+    type: Optional[str] = Query(None, description="Filter by resource type"),
+    region: Optional[str] = Query(None, description="Filter by region"),
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    tag_key: Optional[str] = Query(None, description="Filter by tag key"),
+    tag_value: Optional[str] = Query(None, description="Filter by tag value"),
+    columns: Optional[str] = Query(None, description="Comma-separated column names to include"),
+    sort_by: Optional[str] = Query(None, description="Column to sort by"),
+    sort_order: str = Query("asc", description="Sort order: asc or desc"),
+):
+    """Export resources to CSV file."""
+    store = get_resource_store()
+
+    # Define base columns
+    base_columns = ["name", "arn", "resource_type", "region", "snapshot_name", "tags", "created_at", "config_hash"]
+
+    # Parse requested columns
+    requested_columns = [c.strip() for c in columns.split(",")] if columns else base_columns
+
+    # Check if we need tags (either "tags" column or any "tag:KEY" column)
+    include_tags = any(c == "tags" or c.startswith("tag:") for c in requested_columns)
+
+    # Get all matching resources (no limit for export)
+    resources = store.search(
+        arn_pattern=q,
+        resource_type=type,
+        region=region,
+        snapshot_name=snapshot,
+        tag_key=tag_key,
+        tag_value=tag_value,
+        limit=10000,  # Reasonable limit for export
+        offset=0,
+    )
+
+    if not resources:
+        raise HTTPException(status_code=404, detail="No resources found matching criteria")
+
+    # Enrich resources with tags if needed
+    enriched_resources = []
+    for resource in resources:
+        r = dict(resource)
+        if include_tags:
+            tags = store.get_tags_for_resource(
+                resource["arn"],
+                snapshot_name=resource.get("snapshot_name"),
+            )
+            # "tags" column: all tags as string
+            r["tags"] = "; ".join(f"{k}={v}" for k, v in tags.items()) if tags else ""
+            # Individual tag columns (tag:KEY)
+            for col in requested_columns:
+                if col.startswith("tag:"):
+                    tag_key_name = col[4:]  # Remove "tag:" prefix
+                    r[col] = tags.get(tag_key_name, "")
+        enriched_resources.append(r)
+    resources = enriched_resources
+
+    # Filter columns to only those requested (allow tag:KEY columns)
+    selected_columns = [c for c in requested_columns if c in base_columns or c.startswith("tag:")]
+    if not selected_columns:
+        selected_columns = base_columns
+
+    # Sort if requested
+    if sort_by and (sort_by in base_columns or sort_by.startswith("tag:")):
+        reverse = sort_order.lower() == "desc"
+        resources = sorted(
+            resources,
+            key=lambda r: (r.get(sort_by) or "") if isinstance(r.get(sort_by), str) else r.get(sort_by, 0),
+            reverse=reverse,
+        )
+
+    # Generate CSV
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=selected_columns, extrasaction="ignore")
+    writer.writeheader()
+    writer.writerows(resources)
+
+    # Create response
+    csv_content = output.getvalue()
+    output.close()
+
+    timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    filename = f"resources_export_{timestamp}.csv"
+
+    return StreamingResponse(
+        iter([csv_content]),
+        media_type="text/csv",
+        headers={"Content-Disposition": f"attachment; filename={filename}"},
+    )
+
+
+@router.get("/export/yaml")
+async def export_resources_yaml(
+    q: Optional[str] = Query(None, description="Search query (ARN pattern)"),
+    type: Optional[str] = Query(None, description="Filter by resource type (comma-separated for multiple)"),
+    region: Optional[str] = Query(None, description="Filter by region (comma-separated for multiple)"),
+    snapshot: Optional[str] = Query(None, description="Limit to specific snapshot"),
+    tag_key: Optional[str] = Query(None, description="Filter by tag key"),
+    tag_value: Optional[str] = Query(None, description="Filter by tag value"),
+    include_config: bool = Query(True, description="Include raw configuration"),
+    include_tags: bool = Query(True, description="Include resource tags"),
+):
+    """Export resources to YAML file with all properties."""
+    store = get_resource_store()
+    db = get_database()
+
+    # Parse comma-separated types and regions
+    types_list = [t.strip() for t in type.split(",")] if type else []
+    regions_list = [r.strip() for r in region.split(",")] if region else []
+
+    # For single value, use API filter; for multiple, we'll post-filter
+    api_type = types_list[0] if len(types_list) == 1 else None
+    api_region = regions_list[0] if len(regions_list) == 1 else None
+
+    # Get all matching resources
+    resources = store.search(
+        arn_pattern=q,
+        resource_type=api_type,
+        region=api_region,
+        snapshot_name=snapshot,
+        tag_key=tag_key,
+        tag_value=tag_value,
+        limit=10000,
+        offset=0,
+    )
+
+    # Post-filter for multiple types/regions
+    if len(types_list) > 1:
+        resources = [r for r in resources if r.get("resource_type") in types_list]
+    if len(regions_list) > 1:
+        resources = [r for r in resources if r.get("region") in regions_list]
+
+    if not resources:
+        raise HTTPException(status_code=404, detail="No resources found matching criteria")
+
+    # Enrich resources with full data
+    enriched_resources: List[Dict[str, Any]] = []
+
+    for resource in resources:
+        enriched = dict(resource)
+
+        # Get tags if requested
+        if include_tags:
+            tags = store.get_tags_for_resource(
+                resource["arn"],
+                snapshot_name=resource.get("snapshot_name"),
+            )
+            enriched["tags"] = tags
+
+        # Get raw config if requested
+        if include_config:
+            # Fetch raw_config from database
+            row = db.fetchone(
+                """
+                SELECT r.raw_config
+                FROM resources r
+                JOIN snapshots s ON r.snapshot_id = s.id
+                WHERE r.arn = ? AND s.name = ?
+                """,
+                (resource["arn"], resource.get("snapshot_name")),
+            )
+            if row and row["raw_config"]:
+                try:
+                    enriched["config"] = json.loads(row["raw_config"])
+                except json.JSONDecodeError:
+                    enriched["config"] = row["raw_config"]
+
+        enriched_resources.append(enriched)
+
+    # Generate YAML
+    yaml_content = yaml.dump(
+        {"resources": enriched_resources, "count": len(enriched_resources)},
+        default_flow_style=False,
+        allow_unicode=True,
+        sort_keys=False,
+    )
+
+    timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+    filename = f"resources_export_{timestamp}.yaml"
+
+    return StreamingResponse(
+        iter([yaml_content]),
+        media_type="application/x-yaml",
+        headers={"Content-Disposition": f"attachment; filename={filename}"},
+    )

src/web/routes/api/snapshots.py

@@ -0,0 +1,314 @@
+"""Snapshot API endpoints."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+from fastapi import APIRouter, BackgroundTasks, HTTPException, Query
+from pydantic import BaseModel
+
+from ...dependencies import get_resource_store, get_snapshot_store
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/snapshots")
+
+# In-memory store for snapshot creation jobs
+_snapshot_jobs: Dict[str, dict] = {}
+
+
+class SnapshotSummary(BaseModel):
+    """Snapshot summary for list view."""
+
+    name: str
+    created_at: str
+    account_id: str
+    regions: List[str]
+    resource_count: int
+    is_active: bool
+
+
+class SnapshotDetail(BaseModel):
+    """Full snapshot details."""
+
+    name: str
+    created_at: str
+    account_id: str
+    regions: List[str]
+    resource_count: int
+    service_counts: dict
+    is_active: bool
+    metadata: Optional[dict] = None
+
+
+@router.get("", response_model=List[SnapshotSummary])
+async def list_snapshots():
+    """List all snapshots."""
+    store = get_snapshot_store()
+    snapshots = store.list_all()
+
+    return [
+        SnapshotSummary(
+            name=s["name"],
+            created_at=s["created_at"].isoformat() if hasattr(s["created_at"], "isoformat") else str(s["created_at"]),
+            account_id=s["account_id"],
+            regions=s.get("regions", []),
+            resource_count=s.get("resource_count", 0),
+            is_active=s.get("is_active", False),
+        )
+        for s in snapshots
+    ]
+
+
+class CreateSnapshotRequest(BaseModel):
+    """Request model for creating a snapshot."""
+
+    name: Optional[str] = None  # Auto-generated if not provided
+    regions: Optional[List[str]] = None  # Defaults to us-east-1
+    inventory: Optional[str] = None  # Inventory name to use
+    set_active: bool = True
+    use_config: bool = True  # Use AWS Config for collection
+
+
+class SnapshotJobStatus(BaseModel):
+    """Status of a snapshot creation job."""
+
+    job_id: str
+    status: str  # pending, running, completed, failed
+    snapshot_name: Optional[str] = None
+    message: Optional[str] = None
+    progress: Optional[int] = None
+    created_at: str
+
+
+def _create_snapshot_sync(job_id: str, name: str, regions: List[str], inventory: Optional[str], set_active: bool, use_config: bool):
+    """Synchronous snapshot creation function to run in background."""
+    import boto3
+    import sys
+    import os
+    # Add project root to path for imports in thread
+    project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+
+    from src.config import config
+    from src.snapshot.storage import SnapshotStorage
+    from src.snapshot.inventory_storage import InventoryStorage
+    from src.snapshot.collector import SnapshotCollector
+
+    try:
+        _snapshot_jobs[job_id]["status"] = "running"
+        _snapshot_jobs[job_id]["message"] = "Validating AWS credentials..."
+
+        # Get AWS identity
+        sts = boto3.client("sts")
+        identity = sts.get_caller_identity()
+        account_id = identity["Account"]
+
+        _snapshot_jobs[job_id]["message"] = f"Authenticated as account {account_id}"
+
+        # Load inventory if specified
+        inventory_storage = InventoryStorage(config.storage_path)
+        active_inventory = None
+        include_tags = {}
+        exclude_tags = {}
+
+        if inventory:
+            try:
+                active_inventory = inventory_storage.get_by_name(inventory, account_id)
+                include_tags = active_inventory.include_tags or {}
+                exclude_tags = active_inventory.exclude_tags or {}
+                _snapshot_jobs[job_id]["message"] = f"Using inventory: {inventory}"
+            except Exception:
+                _snapshot_jobs[job_id]["status"] = "failed"
+                _snapshot_jobs[job_id]["message"] = f"Inventory '{inventory}' not found"
+                return
+
+        _snapshot_jobs[job_id]["message"] = f"Collecting resources from {', '.join(regions)}..."
+        _snapshot_jobs[job_id]["progress"] = 10
+
+        # Create collector and collect resources
+        collector = SnapshotCollector(
+            regions=regions,
+            include_tags=include_tags,
+            exclude_tags=exclude_tags,
+            use_config=use_config,
+        )
+
+        snapshot = collector.collect(snapshot_name=name)
+        _snapshot_jobs[job_id]["progress"] = 80
+
+        # Save snapshot
+        _snapshot_jobs[job_id]["message"] = f"Saving snapshot with {snapshot.resource_count} resources..."
+        storage = SnapshotStorage(config.storage_path)
+        snapshot.is_active = set_active
+        storage.save_snapshot(snapshot)
+
+        _snapshot_jobs[job_id]["progress"] = 100
+        _snapshot_jobs[job_id]["status"] = "completed"
+        _snapshot_jobs[job_id]["snapshot_name"] = snapshot.name
+        _snapshot_jobs[job_id]["message"] = f"Created snapshot '{snapshot.name}' with {snapshot.resource_count} resources"
+
+    except Exception as e:
+        logger.exception(f"Snapshot creation failed: {e}")
+        _snapshot_jobs[job_id]["status"] = "failed"
+        _snapshot_jobs[job_id]["message"] = str(e)
+
+
+@router.post("")
+async def create_snapshot(request: CreateSnapshotRequest, background_tasks: BackgroundTasks):
+    """Create a new snapshot (runs in background)."""
+    from datetime import datetime
+
+    # Generate snapshot name if not provided
+    name = request.name
+    if not name:
+        name = f"snapshot-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
+
+    # Default regions
+    regions = request.regions or ["us-east-1"]
+
+    # Create job ID
+    job_id = str(uuid.uuid4())[:8]
+
+    # Initialize job status
+    _snapshot_jobs[job_id] = {
+        "job_id": job_id,
+        "status": "pending",
+        "snapshot_name": name,
+        "message": "Starting snapshot creation...",
+        "progress": 0,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+    }
+
+    # Run in background thread (not async since boto3 is sync)
+    import threading
+    thread = threading.Thread(
+        target=_create_snapshot_sync,
+        args=(job_id, name, regions, request.inventory, request.set_active, request.use_config),
+        daemon=True,
+    )
+    thread.start()
+
+    return {
+        "job_id": job_id,
+        "message": f"Snapshot creation started for '{name}'",
+        "status_url": f"/api/snapshots/jobs/{job_id}",
+    }
+
+
+@router.get("/jobs/{job_id}")
+async def get_snapshot_job_status(job_id: str):
+    """Get status of a snapshot creation job."""
+    if job_id not in _snapshot_jobs:
+        raise HTTPException(status_code=404, detail=f"Job '{job_id}' not found")
+
+    return _snapshot_jobs[job_id]
+
+
+@router.get("/{name}")
+async def get_snapshot(name: str):
+    """Get snapshot details."""
+    store = get_snapshot_store()
+    snapshot = store.load(name)
+
+    if not snapshot:
+        raise HTTPException(status_code=404, detail=f"Snapshot '{name}' not found")
+
+    return {
+        "name": snapshot.name,
+        "created_at": snapshot.created_at.isoformat(),
+        "account_id": snapshot.account_id,
+        "regions": snapshot.regions,
+        "resource_count": snapshot.resource_count,
+        "service_counts": snapshot.service_counts,
+        "is_active": snapshot.is_active,
+        "metadata": snapshot.metadata,
+    }
+
+
+@router.delete("/{name}")
+async def delete_snapshot(name: str):
+    """Delete a snapshot."""
+    store = get_snapshot_store()
+
+    if not store.exists(name):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{name}' not found")
+
+    success = store.delete(name)
+    if not success:
+        raise HTTPException(status_code=500, detail="Failed to delete snapshot")
+
+    return {"message": f"Snapshot '{name}' deleted"}
+
+
+@router.post("/{name}/activate")
+async def activate_snapshot(name: str):
+    """Set snapshot as active baseline."""
+    store = get_snapshot_store()
+
+    if not store.exists(name):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{name}' not found")
+
+    store.set_active(name)
+    return {"message": f"Snapshot '{name}' is now active"}
+
+
+class RenameRequest(BaseModel):
+    """Request model for renaming."""
+
+    new_name: str
+
+
+@router.post("/{name}/rename")
+async def rename_snapshot(name: str, request: RenameRequest):
+    """Rename a snapshot."""
+    store = get_snapshot_store()
+
+    if not store.exists(name):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{name}' not found")
+
+    try:
+        success = store.rename(name, request.new_name)
+        if not success:
+            raise HTTPException(status_code=500, detail="Failed to rename snapshot")
+
+        return {"message": f"Snapshot renamed from '{name}' to '{request.new_name}'", "new_name": request.new_name}
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@router.get("/{name}/resources")
+async def get_snapshot_resources(
+    name: str,
+    type: Optional[str] = Query(None, description="Filter by resource type"),
+    region: Optional[str] = Query(None, description="Filter by region"),
+    limit: int = Query(100, le=1000),
+    offset: int = Query(0, ge=0),
+):
+    """Get resources in a snapshot."""
+    snapshot_store = get_snapshot_store()
+
+    if not snapshot_store.exists(name):
+        raise HTTPException(status_code=404, detail=f"Snapshot '{name}' not found")
+
+    resource_store = get_resource_store()
+    resources = resource_store.search(
+        snapshot_name=name,
+        resource_type=type,
+        region=region,
+        limit=limit,
+        offset=offset,
+    )
+
+    return {
+        "snapshot": name,
+        "count": len(resources),
+        "limit": limit,
+        "offset": offset,
+        "resources": resources,
+    }