aws-inventory-manager 0.17.12__py3-none-any.whl

src/storage/audit_store.py

@@ -0,0 +1,419 @@
+"""Audit storage operations for SQLite backend."""
+
+import logging
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from ..models.deletion_operation import DeletionOperation, OperationMode, OperationStatus
+from ..models.deletion_record import DeletionRecord, DeletionStatus
+from .database import Database, json_deserialize, json_serialize
+
+logger = logging.getLogger(__name__)
+
+
+class AuditStore:
+    """CRUD operations for audit logs in SQLite database."""
+
+    def __init__(self, db: Database):
+        """Initialize audit store.
+
+        Args:
+            db: Database connection manager
+        """
+        self.db = db
+
+    def save_operation(self, operation: DeletionOperation) -> str:
+        """Save or update deletion operation.
+
+        Args:
+            operation: DeletionOperation to save
+
+        Returns:
+            Operation ID
+        """
+        with self.db.transaction() as cursor:
+            # Check if operation exists
+            existing = self.db.fetchone(
+                "SELECT operation_id FROM audit_operations WHERE operation_id = ?",
+                (operation.operation_id,),
+            )
+
+            if existing:
+                # Update existing
+                cursor.execute(
+                    """
+                    UPDATE audit_operations SET
+                        status = ?,
+                        succeeded_count = ?,
+                        failed_count = ?,
+                        skipped_count = ?,
+                        duration_seconds = ?
+                    WHERE operation_id = ?
+                    """,
+                    (
+                        operation.status.value,
+                        operation.succeeded_count,
+                        operation.failed_count,
+                        operation.skipped_count,
+                        operation.duration_seconds,
+                        operation.operation_id,
+                    ),
+                )
+                logger.debug(f"Updated audit operation '{operation.operation_id}'")
+            else:
+                # Insert new
+                cursor.execute(
+                    """
+                    INSERT INTO audit_operations (
+                        operation_id, baseline_snapshot, timestamp, aws_profile,
+                        account_id, mode, status, total_resources, succeeded_count,
+                        failed_count, skipped_count, duration_seconds, filters
+                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """,
+                    (
+                        operation.operation_id,
+                        operation.baseline_snapshot,
+                        operation.timestamp.isoformat(),
+                        operation.aws_profile,
+                        operation.account_id,
+                        operation.mode.value,
+                        operation.status.value,
+                        operation.total_resources,
+                        operation.succeeded_count,
+                        operation.failed_count,
+                        operation.skipped_count,
+                        operation.duration_seconds,
+                        json_serialize(operation.filters),
+                    ),
+                )
+                logger.debug(f"Saved audit operation '{operation.operation_id}'")
+
+        return operation.operation_id
+
+    def save_record(self, record: DeletionRecord) -> str:
+        """Save deletion record.
+
+        Args:
+            record: DeletionRecord to save
+
+        Returns:
+            Record ID
+        """
+        with self.db.transaction() as cursor:
+            cursor.execute(
+                """
+                INSERT INTO audit_records (
+                    operation_id, resource_arn, resource_id, resource_type,
+                    region, status, error_code, error_message, protection_reason,
+                    deletion_tier, tags, estimated_monthly_cost
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    record.operation_id,
+                    record.resource_arn,
+                    record.resource_id,
+                    record.resource_type,
+                    record.region,
+                    record.status.value,
+                    record.error_code,
+                    record.error_message,
+                    record.protection_reason,
+                    record.deletion_tier,
+                    json_serialize(record.tags),
+                    record.estimated_monthly_cost,
+                ),
+            )
+
+        logger.debug(f"Saved audit record for '{record.resource_arn}'")
+        return record.record_id
+
+    def save_records_batch(self, records: List[DeletionRecord]) -> int:
+        """Save multiple deletion records efficiently.
+
+        Args:
+            records: List of DeletionRecord objects
+
+        Returns:
+            Number of records saved
+        """
+        if not records:
+            return 0
+
+        with self.db.transaction() as cursor:
+            data = [
+                (
+                    r.operation_id,
+                    r.resource_arn,
+                    r.resource_id,
+                    r.resource_type,
+                    r.region,
+                    r.status.value,
+                    r.error_code,
+                    r.error_message,
+                    r.protection_reason,
+                    r.deletion_tier,
+                    json_serialize(r.tags),
+                    r.estimated_monthly_cost,
+                )
+                for r in records
+            ]
+            cursor.executemany(
+                """
+                INSERT INTO audit_records (
+                    operation_id, resource_arn, resource_id, resource_type,
+                    region, status, error_code, error_message, protection_reason,
+                    deletion_tier, tags, estimated_monthly_cost
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                data,
+            )
+
+        logger.debug(f"Saved {len(records)} audit records")
+        return len(records)
+
+    def load_operation(self, operation_id: str) -> Optional[DeletionOperation]:
+        """Load deletion operation by ID.
+
+        Args:
+            operation_id: Operation ID
+
+        Returns:
+            DeletionOperation or None if not found
+        """
+        row = self.db.fetchone(
+            "SELECT * FROM audit_operations WHERE operation_id = ?",
+            (operation_id,),
+        )
+        if not row:
+            return None
+
+        return self._row_to_operation(row)
+
+    def _row_to_operation(self, row: Dict[str, Any]) -> DeletionOperation:
+        """Convert database row to DeletionOperation.
+
+        Args:
+            row: Database row dict
+
+        Returns:
+            DeletionOperation object
+        """
+        timestamp = datetime.fromisoformat(row["timestamp"])
+        if timestamp.tzinfo is None:
+            timestamp = timestamp.replace(tzinfo=timezone.utc)
+
+        return DeletionOperation(
+            operation_id=row["operation_id"],
+            baseline_snapshot=row["baseline_snapshot"],
+            timestamp=timestamp,
+            aws_profile=row["aws_profile"],
+            account_id=row["account_id"],
+            mode=OperationMode(row["mode"]),
+            status=OperationStatus(row["status"]),
+            total_resources=row["total_resources"],
+            succeeded_count=row["succeeded_count"] or 0,
+            failed_count=row["failed_count"] or 0,
+            skipped_count=row["skipped_count"] or 0,
+            duration_seconds=row["duration_seconds"],
+            filters=json_deserialize(row["filters"]),
+        )
+
+    def load_records(self, operation_id: str) -> List[DeletionRecord]:
+        """Load all records for an operation.
+
+        Args:
+            operation_id: Operation ID
+
+        Returns:
+            List of DeletionRecord objects
+        """
+        rows = self.db.fetchall(
+            "SELECT * FROM audit_records WHERE operation_id = ? ORDER BY id",
+            (operation_id,),
+        )
+        return [self._row_to_record(row) for row in rows]
+
+    def _row_to_record(self, row: Dict[str, Any]) -> DeletionRecord:
+        """Convert database row to DeletionRecord.
+
+        Args:
+            row: Database row dict
+
+        Returns:
+            DeletionRecord object
+        """
+        return DeletionRecord(
+            record_id=str(row["id"]),
+            operation_id=row["operation_id"],
+            resource_arn=row["resource_arn"],
+            resource_id=row["resource_id"] or "",
+            resource_type=row["resource_type"],
+            region=row["region"],
+            timestamp=datetime.now(timezone.utc),  # Not stored in DB, use current time
+            status=DeletionStatus(row["status"]),
+            error_code=row["error_code"],
+            error_message=row["error_message"],
+            protection_reason=row["protection_reason"],
+            deletion_tier=row["deletion_tier"],
+            tags=json_deserialize(row["tags"]),
+            estimated_monthly_cost=row["estimated_monthly_cost"],
+        )
+
+    def list_operations(
+        self,
+        account_id: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[DeletionOperation]:
+        """List audit operations.
+
+        Args:
+            account_id: Filter by account ID (optional)
+            limit: Maximum results
+
+        Returns:
+            List of DeletionOperation objects
+        """
+        if account_id:
+            rows = self.db.fetchall(
+                "SELECT * FROM audit_operations WHERE account_id = ? ORDER BY timestamp DESC LIMIT ?",
+                (account_id, limit),
+            )
+        else:
+            rows = self.db.fetchall(
+                "SELECT * FROM audit_operations ORDER BY timestamp DESC LIMIT ?",
+                (limit,),
+            )
+
+        return [self._row_to_operation(row) for row in rows]
+
+    def delete_operation(self, operation_id: str) -> bool:
+        """Delete operation and all its records.
+
+        Args:
+            operation_id: Operation ID
+
+        Returns:
+            True if deleted, False if not found
+        """
+        with self.db.transaction() as cursor:
+            # Records are deleted by CASCADE
+            cursor.execute(
+                "DELETE FROM audit_operations WHERE operation_id = ?",
+                (operation_id,),
+            )
+            deleted = cursor.rowcount > 0
+
+        if deleted:
+            logger.debug(f"Deleted audit operation '{operation_id}'")
+        return deleted
+
+    def get_operation_stats(self, operation_id: str) -> Dict[str, Any]:
+        """Get statistics for an operation.
+
+        Args:
+            operation_id: Operation ID
+
+        Returns:
+            Dictionary with statistics
+        """
+        operation = self.load_operation(operation_id)
+        if not operation:
+            return {}
+
+        # Get record counts by status
+        status_counts = self.db.fetchall(
+            """
+            SELECT status, COUNT(*) as count
+            FROM audit_records
+            WHERE operation_id = ?
+            GROUP BY status
+            """,
+            (operation_id,),
+        )
+
+        # Get counts by resource type
+        type_counts = self.db.fetchall(
+            """
+            SELECT resource_type, COUNT(*) as count
+            FROM audit_records
+            WHERE operation_id = ?
+            GROUP BY resource_type
+            ORDER BY count DESC
+            """,
+            (operation_id,),
+        )
+
+        # Get total estimated cost
+        cost_row = self.db.fetchone(
+            """
+            SELECT SUM(estimated_monthly_cost) as total_cost
+            FROM audit_records
+            WHERE operation_id = ?
+            """,
+            (operation_id,),
+        )
+
+        return {
+            "operation_id": operation_id,
+            "baseline_snapshot": operation.baseline_snapshot,
+            "mode": operation.mode.value,
+            "status": operation.status.value,
+            "total_resources": operation.total_resources,
+            "succeeded_count": operation.succeeded_count,
+            "failed_count": operation.failed_count,
+            "skipped_count": operation.skipped_count,
+            "duration_seconds": operation.duration_seconds,
+            "status_breakdown": {row["status"]: row["count"] for row in status_counts},
+            "type_breakdown": {row["resource_type"]: row["count"] for row in type_counts},
+            "total_estimated_monthly_cost": cost_row["total_cost"] if cost_row else 0,
+        }
+
+    def get_recent_deletions(
+        self,
+        resource_arn: Optional[str] = None,
+        resource_type: Optional[str] = None,
+        region: Optional[str] = None,
+        limit: int = 100,
+    ) -> List[Dict[str, Any]]:
+        """Get recent deletion records with filters.
+
+        Args:
+            resource_arn: Filter by ARN pattern (optional)
+            resource_type: Filter by resource type (optional)
+            region: Filter by region (optional)
+            limit: Maximum results
+
+        Returns:
+            List of deletion records with operation info
+        """
+        conditions = []
+        params: List[Any] = []
+
+        if resource_arn:
+            conditions.append("r.resource_arn LIKE ?")
+            params.append(f"%{resource_arn}%")
+
+        if resource_type:
+            conditions.append("r.resource_type LIKE ?")
+            params.append(f"%{resource_type}%")
+
+        if region:
+            conditions.append("r.region = ?")
+            params.append(region)
+
+        where_clause = " AND ".join(conditions) if conditions else "1=1"
+        params.append(limit)
+
+        rows = self.db.fetchall(
+            f"""
+            SELECT r.*, o.timestamp as operation_timestamp, o.mode, o.baseline_snapshot
+            FROM audit_records r
+            JOIN audit_operations o ON r.operation_id = o.operation_id
+            WHERE {where_clause}
+            ORDER BY o.timestamp DESC
+            LIMIT ?
+            """,
+            tuple(params),
+        )
+
+        return [dict(row) for row in rows]

src/storage/database.py

@@ -0,0 +1,294 @@
+"""SQLite database connection and management for AWS Inventory Manager."""
+
+import json
+import logging
+import sqlite3
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Dict, Generator, List, Optional
+
+from ..utils.paths import get_snapshot_storage_path
+from .schema import INDEXES_SQL, SCHEMA_SQL, SCHEMA_VERSION, get_migrations
+
+logger = logging.getLogger(__name__)
+
+
+class Database:
+    """SQLite database connection manager.
+
+    Handles connection pooling, schema setup, and auto-migration from YAML.
+    """
+
+    def __init__(self, db_path: Optional[Path] = None, storage_path: Optional[Path] = None):
+        """Initialize database manager.
+
+        Args:
+            db_path: Direct path to database file (overrides storage_path)
+            storage_path: Storage directory (database will be inventory.db inside)
+        """
+        if db_path:
+            self.db_path = Path(db_path)
+        elif storage_path:
+            self.db_path = Path(storage_path) / "inventory.db"
+        else:
+            self.db_path = get_snapshot_storage_path() / "inventory.db"
+
+        self.storage_path = self.db_path.parent
+        self._connection: Optional[sqlite3.Connection] = None
+        self._initialized = False
+
+    def connect(self) -> sqlite3.Connection:
+        """Get or create database connection with optimal settings.
+
+        Returns:
+            SQLite connection configured for optimal performance
+        """
+        if self._connection is None:
+            # Ensure directory exists
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
+
+            self._connection = sqlite3.connect(str(self.db_path))
+            # Use Row factory for dict-like access
+            self._connection.row_factory = sqlite3.Row
+
+            # Enable foreign keys
+            self._connection.execute("PRAGMA foreign_keys = ON")
+
+            # Use WAL mode for better concurrent performance
+            self._connection.execute("PRAGMA journal_mode = WAL")
+
+            # Optimize for performance
+            self._connection.execute("PRAGMA synchronous = NORMAL")
+            self._connection.execute("PRAGMA cache_size = -64000")  # 64MB cache
+            self._connection.execute("PRAGMA temp_store = MEMORY")  # Temp tables in memory
+            self._connection.execute("PRAGMA mmap_size = 268435456")  # 256MB memory-mapped I/O
+
+            logger.debug(f"Connected to database: {self.db_path}")
+
+        return self._connection
+
+    def ensure_schema(self) -> None:
+        """Create database schema if not exists and run migrations."""
+        if self._initialized:
+            return
+
+        conn = self.connect()
+        cursor = conn.cursor()
+
+        try:
+            # Get current schema version before creating tables
+            current_version = self._get_raw_schema_version(cursor)
+
+            # Create tables
+            cursor.executescript(SCHEMA_SQL)
+
+            # Run migrations BEFORE creating indexes (migrations may add columns that indexes depend on)
+            if current_version and current_version != SCHEMA_VERSION:
+                self._run_migrations(cursor, current_version)
+
+            # Create indexes (after migrations so new columns exist)
+            cursor.executescript(INDEXES_SQL)
+
+            # Set schema version
+            cursor.execute(
+                "INSERT OR REPLACE INTO schema_info (key, value) VALUES (?, ?)",
+                ("schema_version", SCHEMA_VERSION),
+            )
+
+            conn.commit()
+            self._initialized = True
+            logger.debug(f"Database schema initialized (version {SCHEMA_VERSION})")
+
+        except sqlite3.Error as e:
+            conn.rollback()
+            logger.error(f"Failed to initialize schema: {e}")
+            raise
+
+    def _get_raw_schema_version(self, cursor: sqlite3.Cursor) -> Optional[str]:
+        """Get schema version without ensuring schema exists.
+
+        Args:
+            cursor: Database cursor
+
+        Returns:
+            Schema version string or None if not set
+        """
+        try:
+            cursor.execute("SELECT value FROM schema_info WHERE key = ?", ("schema_version",))
+            row = cursor.fetchone()
+            return row["value"] if row else None
+        except sqlite3.OperationalError:
+            return None
+
+    def _run_migrations(self, cursor: sqlite3.Cursor, from_version: str) -> None:
+        """Run schema migrations from a given version.
+
+        Args:
+            cursor: Database cursor
+            from_version: Version to migrate from
+        """
+        migrations = get_migrations()
+
+        # Simple version comparison - assumes semantic versioning
+        for version, statements in sorted(migrations.items()):
+            if version > from_version:
+                logger.info(f"Running migration to version {version}")
+                for sql in statements:
+                    try:
+                        cursor.execute(sql)
+                        logger.debug(f"Migration SQL executed: {sql[:50]}...")
+                    except sqlite3.OperationalError as e:
+                        # Column may already exist if migration was partially applied
+                        if "duplicate column name" in str(e).lower():
+                            logger.debug(f"Column already exists, skipping: {e}")
+                        else:
+                            raise
+
+    def get_schema_version(self) -> Optional[str]:
+        """Get current schema version from database.
+
+        Returns:
+            Schema version string or None if not set
+        """
+        conn = self.connect()
+        try:
+            cursor = conn.execute("SELECT value FROM schema_info WHERE key = ?", ("schema_version",))
+            row = cursor.fetchone()
+            return row["value"] if row else None
+        except sqlite3.OperationalError:
+            return None
+
+    def is_empty(self) -> bool:
+        """Check if database has no snapshots.
+
+        Returns:
+            True if no snapshots exist
+        """
+        conn = self.connect()
+        self.ensure_schema()
+        cursor = conn.execute("SELECT COUNT(*) as count FROM snapshots")
+        row = cursor.fetchone()
+        return row["count"] == 0
+
+    @contextmanager
+    def transaction(self) -> Generator[sqlite3.Cursor, None, None]:
+        """Context manager for database transactions.
+
+        Yields:
+            Database cursor
+
+        Example:
+            with db.transaction() as cursor:
+                cursor.execute("INSERT INTO ...")
+        """
+        conn = self.connect()
+        self.ensure_schema()
+        cursor = conn.cursor()
+        try:
+            yield cursor
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+
+    def execute(self, sql: str, params: tuple = ()) -> sqlite3.Cursor:
+        """Execute a single SQL statement.
+
+        Args:
+            sql: SQL statement
+            params: Query parameters
+
+        Returns:
+            Cursor with results
+        """
+        conn = self.connect()
+        self.ensure_schema()
+        return conn.execute(sql, params)
+
+    def executemany(self, sql: str, params_list: List[tuple]) -> sqlite3.Cursor:
+        """Execute SQL statement with multiple parameter sets.
+
+        Args:
+            sql: SQL statement with placeholders
+            params_list: List of parameter tuples
+
+        Returns:
+            Cursor
+        """
+        conn = self.connect()
+        self.ensure_schema()
+        return conn.executemany(sql, params_list)
+
+    def fetchall(self, sql: str, params: tuple = ()) -> List[Dict[str, Any]]:
+        """Execute query and fetch all results as dicts.
+
+        Args:
+            sql: SQL query
+            params: Query parameters
+
+        Returns:
+            List of result dictionaries
+        """
+        cursor = self.execute(sql, params)
+        return [dict(row) for row in cursor.fetchall()]
+
+    def fetchone(self, sql: str, params: tuple = ()) -> Optional[Dict[str, Any]]:
+        """Execute query and fetch single result as dict.
+
+        Args:
+            sql: SQL query
+            params: Query parameters
+
+        Returns:
+            Result dictionary or None
+        """
+        cursor = self.execute(sql, params)
+        row = cursor.fetchone()
+        return dict(row) if row else None
+
+    def close(self) -> None:
+        """Close database connection."""
+        if self._connection:
+            self._connection.close()
+            self._connection = None
+            self._initialized = False
+            logger.debug("Database connection closed")
+
+    def __enter__(self) -> "Database":
+        """Context manager entry."""
+        self.connect()
+        self.ensure_schema()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        """Context manager exit."""
+        self.close()
+
+
+# JSON serialization helpers for SQLite
+def json_serialize(obj: Any) -> Optional[str]:
+    """Serialize object to JSON string for SQLite storage.
+
+    Args:
+        obj: Object to serialize
+
+    Returns:
+        JSON string or None if obj is None
+    """
+    if obj is None:
+        return None
+    return json.dumps(obj, default=str)
+
+
+def json_deserialize(json_str: Optional[str]) -> Any:
+    """Deserialize JSON string from SQLite.
+
+    Args:
+        json_str: JSON string
+
+    Returns:
+        Deserialized object or None
+    """
+    if json_str is None:
+        return None
+    return json.loads(json_str)