aws-inventory-manager 0.13.2__py3-none-any.whl

src/snapshot/filter.py

@@ -0,0 +1,259 @@
+"""Resource filtering for creating historical and tagged baselines."""
+
+import logging
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from ..models.resource import Resource
+
+logger = logging.getLogger(__name__)
+
+
+class ResourceFilter:
+    """Filter resources by creation date and tags."""
+
+    def __init__(
+        self,
+        before_date: Optional[datetime] = None,
+        after_date: Optional[datetime] = None,
+        required_tags: Optional[Dict[str, str]] = None,
+        include_tags: Optional[Dict[str, str]] = None,
+        exclude_tags: Optional[Dict[str, str]] = None,
+    ):
+        """Initialize resource filter.
+
+        Args:
+            before_date: Include only resources created before this date (exclusive)
+            after_date: Include only resources created on or after this date (inclusive)
+            required_tags: DEPRECATED - use include_tags instead (kept for backward compatibility)
+            include_tags: Resources must have ALL these tags (AND logic)
+            exclude_tags: Resources must NOT have ANY of these tags (OR logic)
+        """
+        self.before_date = before_date
+        self.after_date = after_date
+        # Support both required_tags (deprecated) and include_tags (new)
+        self.include_tags = include_tags or required_tags or {}
+        self.exclude_tags = exclude_tags or {}
+        # Keep required_tags for backward compatibility
+        self.required_tags = self.include_tags
+
+        # Statistics
+        self.stats = {
+            "total_collected": 0,
+            "date_matched": 0,
+            "tag_matched": 0,
+            "final_count": 0,
+            "filtered_out_by_date": 0,
+            "filtered_out_by_tags": 0,
+            "filtered_out_by_exclude_tags": 0,
+            "missing_creation_date": 0,
+        }
+
+    def apply(self, resources: List[Resource]) -> List[Resource]:
+        """Apply filters to a list of resources.
+
+        Args:
+            resources: List of resources to filter
+
+        Returns:
+            Filtered list of resources
+        """
+        self.stats["total_collected"] = len(resources)
+        filtered = []
+
+        for resource in resources:
+            if self._matches_filters(resource):
+                filtered.append(resource)
+
+        self.stats["final_count"] = len(filtered)
+
+        logger.debug(
+            f"Filtering complete: {self.stats['total_collected']} collected, "
+            f"{self.stats['final_count']} matched filters"
+        )
+
+        return filtered
+
+    def _matches_filters(self, resource: Resource) -> bool:
+        """Check if a resource matches all filters.
+
+        Args:
+            resource: Resource to check
+
+        Returns:
+            True if resource matches all filters
+        """
+        # Check date filters
+        if not self._matches_date_filter(resource):
+            self.stats["filtered_out_by_date"] += 1
+            return False
+
+        self.stats["date_matched"] += 1
+
+        # Check exclude tags first (if resource has any excluded tags, reject immediately)
+        if not self._matches_exclude_filter(resource):
+            self.stats["filtered_out_by_exclude_tags"] += 1
+            return False
+
+        # Check include tag filters
+        if not self._matches_tag_filter(resource):
+            self.stats["filtered_out_by_tags"] += 1
+            return False
+
+        self.stats["tag_matched"] += 1
+
+        return True
+
+    def _matches_date_filter(self, resource: Resource) -> bool:
+        """Check if resource matches date filters.
+
+        Args:
+            resource: Resource to check
+
+        Returns:
+            True if resource matches date filters (or no date filters specified)
+        """
+        # If no date filters, everything matches
+        if not self.before_date and not self.after_date:
+            return True
+
+        # If resource has no creation date or invalid type, we can't filter by date
+        # Many AWS resources don't expose creation timestamps (VPCs, Security Groups, etc.)
+        if not resource.created_at:
+            self.stats["missing_creation_date"] += 1
+            logger.debug(f"Resource {resource.arn} has no creation date - including by default")
+            return True
+
+        # Handle string dates (convert to datetime)
+        resource_date = resource.created_at
+        if isinstance(resource_date, str):
+            try:
+                resource_date = datetime.fromisoformat(resource_date.replace("Z", "+00:00"))
+            except ValueError:
+                self.stats["missing_creation_date"] += 1
+                logger.debug(f"Resource {resource.arn} has invalid date format - including by default")
+                return True
+
+        # Ensure we have a datetime object
+        if not isinstance(resource_date, datetime):
+            self.stats["missing_creation_date"] += 1
+            logger.debug(f"Resource {resource.arn} has no valid creation date - including by default")
+            return True
+
+        # Make sure resource.created_at is timezone-aware for comparison
+        if resource_date.tzinfo is None:
+            # Assume UTC if no timezone
+            from datetime import timezone as tz
+
+            resource_date = resource_date.replace(tzinfo=tz.utc)
+
+        # Check before_date filter (exclusive)
+        if self.before_date:
+            # Make sure before_date is timezone-aware
+            before_date_aware = self.before_date
+            if before_date_aware.tzinfo is None:
+                from datetime import timezone as tz
+
+                before_date_aware = before_date_aware.replace(tzinfo=tz.utc)
+
+            if resource_date >= before_date_aware:
+                logger.debug(f"Resource {resource.name} created {resource_date} " f"is not before {before_date_aware}")
+                return False
+
+        # Check after_date filter (inclusive)
+        if self.after_date:
+            # Make sure after_date is timezone-aware
+            after_date_aware = self.after_date
+            if after_date_aware.tzinfo is None:
+                from datetime import timezone as tz
+
+                after_date_aware = after_date_aware.replace(tzinfo=tz.utc)
+
+            if resource_date < after_date_aware:
+                logger.debug(f"Resource {resource.name} created {resource_date} " f"is before {after_date_aware}")
+                return False
+
+        return True
+
+    def _matches_tag_filter(self, resource: Resource) -> bool:
+        """Check if resource has all include tags (AND logic).
+
+        Args:
+            resource: Resource to check
+
+        Returns:
+            True if resource has all include tags (or no tag filters specified)
+        """
+        # If no include tag filters, everything matches
+        if not self.include_tags:
+            return True
+
+        # Check if resource has ALL include tags with matching values (AND logic)
+        for key, value in self.include_tags.items():
+            if key not in resource.tags:
+                logger.debug(f"Resource {resource.name} missing include tag: {key}")
+                return False
+
+            if resource.tags[key] != value:
+                logger.debug(
+                    f"Resource {resource.name} tag {key}={resource.tags[key]} " f"does not match required value {value}"
+                )
+                return False
+
+        return True
+
+    def _matches_exclude_filter(self, resource: Resource) -> bool:
+        """Check if resource has any exclude tags (OR logic).
+
+        Args:
+            resource: Resource to check
+
+        Returns:
+            True if resource does NOT have any exclude tags (or no exclude filters specified)
+        """
+        # If no exclude tag filters, everything matches
+        if not self.exclude_tags:
+            return True
+
+        # Check if resource has ANY of the exclude tags (OR logic)
+        for key, value in self.exclude_tags.items():
+            if key in resource.tags and resource.tags[key] == value:
+                logger.debug(f"Resource {resource.name} has exclude tag {key}={value}")
+                return False
+
+        return True
+
+    def get_filter_summary(self) -> str:
+        """Get a human-readable summary of applied filters.
+
+        Returns:
+            Formatted string describing the filters
+        """
+        parts = []
+
+        if self.before_date:
+            parts.append(f"created before {self.before_date.strftime('%Y-%m-%d')}")
+
+        if self.after_date:
+            parts.append(f"created on/after {self.after_date.strftime('%Y-%m-%d')}")
+
+        if self.include_tags:
+            tag_strs = [f"{k}={v}" for k, v in self.include_tags.items()]
+            parts.append(f"include tags: {', '.join(tag_strs)}")
+
+        if self.exclude_tags:
+            tag_strs = [f"{k}={v}" for k, v in self.exclude_tags.items()]
+            parts.append(f"exclude tags: {', '.join(tag_strs)}")
+
+        if not parts:
+            return "No filters applied"
+
+        return "Filters: " + " AND ".join(parts)
+
+    def get_statistics_summary(self) -> Dict[str, int]:
+        """Get filtering statistics.
+
+        Returns:
+            Dictionary of filtering statistics
+        """
+        return self.stats.copy()

src/snapshot/inventory_storage.py

@@ -0,0 +1,236 @@
+"""Storage service for inventory management.
+
+This module provides the main interface for inventory persistence.
+It uses SQLite as the primary storage backend, with automatic migration
+from legacy YAML files on first use.
+"""
+
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional, Union
+
+import yaml
+
+from ..models.inventory import Inventory
+from ..storage import Database, InventoryStore
+from ..utils.paths import get_snapshot_storage_path
+
+logger = logging.getLogger(__name__)
+
+
+class InventoryNotFoundError(Exception):
+    """Raised when an inventory cannot be found."""
+
+    pass
+
+
+class InventoryStorage:
+    """Manage inventory storage and retrieval using SQLite backend.
+
+    Handles CRUD operations for inventories with automatic migration
+    from legacy YAML files on first use.
+    """
+
+    def __init__(self, storage_dir: Optional[Union[str, Path]] = None):
+        """Initialize inventory storage.
+
+        Args:
+            storage_dir: Directory for storage (default: ~/.snapshots via get_snapshot_storage_path())
+        """
+        self.storage_dir = get_snapshot_storage_path(storage_dir)
+        self.inventory_file = self.storage_dir / "inventories.yaml"
+
+        # Ensure storage directory exists
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+
+        # Initialize SQLite database
+        self.db = Database(storage_path=self.storage_dir)
+        self.db.ensure_schema()
+
+        # Initialize inventory store
+        self._store = InventoryStore(self.db)
+
+        # Auto-migrate YAML inventories on first use
+        self._migrate_yaml_if_needed()
+
+    def _migrate_yaml_if_needed(self) -> None:
+        """Migrate inventories from YAML to SQLite if needed."""
+        if not self.inventory_file.exists():
+            return
+
+        # Check if we have inventories in SQLite already
+        existing = self._store.list_all()
+        if existing:
+            return  # Already migrated
+
+        try:
+            with open(self.inventory_file, "r") as f:
+                data = yaml.safe_load(f)
+
+            if not data or "inventories" not in data:
+                return
+
+            for inv_data in data["inventories"]:
+                inventory = Inventory.from_dict(inv_data)
+                self._store.save(inventory)
+                logger.debug(f"Migrated inventory '{inventory.name}' to SQLite")
+
+            logger.info(f"Migrated {len(data['inventories'])} inventories from YAML to SQLite")
+
+        except Exception as e:
+            logger.warning(f"Failed to migrate YAML inventories: {e}")
+
+    def load_all(self) -> List[Inventory]:
+        """Load all inventories.
+
+        Returns:
+            List of all inventories (empty list if none exist)
+        """
+        inventories = self._store.list_all()
+        logger.debug(f"Loaded {len(inventories)} inventories from storage")
+        return inventories
+
+    def load_by_account(self, account_id: str) -> List[Inventory]:
+        """Load inventories for specific account.
+
+        Args:
+            account_id: AWS account ID (12 digits)
+
+        Returns:
+            List of inventories for the account
+        """
+        inventories = self._store.list_by_account(account_id)
+        logger.debug(f"Found {len(inventories)} inventories for account {account_id}")
+        return inventories
+
+    def get_by_name(self, name: str, account_id: str) -> Inventory:
+        """Get specific inventory by name and account.
+
+        Args:
+            name: Inventory name
+            account_id: AWS account ID
+
+        Returns:
+            Inventory instance
+
+        Raises:
+            InventoryNotFoundError: If inventory not found
+        """
+        inventory = self._store.load(name, account_id)
+        if inventory:
+            logger.debug(f"Found inventory '{name}' for account {account_id}")
+            return inventory
+
+        raise InventoryNotFoundError(f"Inventory '{name}' not found for account {account_id}")
+
+    def get_or_create_default(self, account_id: str) -> Inventory:
+        """Get default inventory, creating if it doesn't exist.
+
+        Args:
+            account_id: AWS account ID
+
+        Returns:
+            Default inventory instance
+        """
+        try:
+            return self.get_by_name("default", account_id)
+        except InventoryNotFoundError:
+            # Auto-create default inventory
+            default = Inventory(
+                name="default",
+                account_id=account_id,
+                description="Auto-created default inventory",
+                include_tags={},
+                exclude_tags={},
+                snapshots=[],
+                active_snapshot=None,
+                created_at=datetime.now(timezone.utc),
+                last_updated=datetime.now(timezone.utc),
+            )
+            self.save(default)
+            logger.info(f"Created default inventory for account {account_id}")
+            return default
+
+    def save(self, inventory: Inventory) -> None:
+        """Save/update inventory.
+
+        Args:
+            inventory: Inventory to save
+
+        Raises:
+            ValueError: If inventory validation fails
+        """
+        # Validate inventory before saving
+        errors = inventory.validate()
+        if errors:
+            raise ValueError(f"Invalid inventory: {', '.join(errors)}")
+
+        self._store.save(inventory)
+        logger.debug(f"Saved inventory '{inventory.name}' for account {inventory.account_id}")
+
+    def delete(self, name: str, account_id: str, delete_snapshots: bool = False) -> int:
+        """Delete inventory, optionally deleting its snapshot files.
+
+        Args:
+            name: Inventory name
+            account_id: AWS account ID
+            delete_snapshots: Whether to delete snapshot files
+
+        Returns:
+            Number of snapshot files deleted (0 if delete_snapshots=False)
+
+        Raises:
+            InventoryNotFoundError: If inventory not found
+        """
+        # Load inventory to get snapshot list
+        inventory = self.get_by_name(name, account_id)
+
+        # Delete snapshot files if requested
+        deleted_count = 0
+        if delete_snapshots:
+            from ..storage import SnapshotStore
+
+            snapshot_store = SnapshotStore(self.db)
+            for snapshot_name in inventory.snapshots:
+                # Remove file extensions if present
+                snap_name = snapshot_name.replace(".yaml.gz", "").replace(".yaml", "")
+                if snapshot_store.delete(snap_name):
+                    deleted_count += 1
+                    logger.debug(f"Deleted snapshot: {snap_name}")
+
+        # Delete inventory
+        self._store.delete(name, account_id)
+        logger.info(f"Deleted inventory '{name}' for account {account_id}")
+
+        return deleted_count
+
+    def exists(self, name: str, account_id: str) -> bool:
+        """Check if inventory exists.
+
+        Args:
+            name: Inventory name
+            account_id: AWS account ID
+
+        Returns:
+            True if inventory exists, False otherwise
+        """
+        return self._store.exists(name, account_id)
+
+    def validate_unique(self, name: str, account_id: str) -> bool:
+        """Validate that (name, account_id) combination is unique.
+
+        Args:
+            name: Inventory name
+            account_id: AWS account ID
+
+        Returns:
+            True if unique, False if already exists
+        """
+        return not self.exists(name, account_id)
+
+    # Legacy methods for backward compatibility
+    def _atomic_write(self, inventories: List[Inventory]) -> None:
+        """Write inventories using atomic rename pattern (legacy, no-op for SQLite)."""
+        pass  # SQLite handles atomicity