aws-inventory-manager 0.17.12__py3-none-any.whl

src/snapshot/resource_collectors/glue.py

@@ -0,0 +1,199 @@
+"""AWS Glue resource collector."""
+
+from typing import List
+
+from ...models.resource import Resource
+from ...utils.hash import compute_config_hash
+from .base import BaseResourceCollector
+
+
+class GlueCollector(BaseResourceCollector):
+    """Collector for AWS Glue resources (databases, tables, crawlers, jobs)."""
+
+    @property
+    def service_name(self) -> str:
+        return "glue"
+
+    def collect(self) -> List[Resource]:
+        """Collect AWS Glue resources.
+
+        Returns:
+            List of Glue resources (databases, tables, crawlers, jobs)
+        """
+        resources = []
+        client = self._create_client()
+        account_id = self._get_account_id()
+
+        # Collect databases and tables
+        resources.extend(self._collect_databases(client, account_id))
+
+        # Collect crawlers
+        resources.extend(self._collect_crawlers(client, account_id))
+
+        # Collect jobs
+        resources.extend(self._collect_jobs(client, account_id))
+
+        # Collect connections
+        resources.extend(self._collect_connections(client, account_id))
+
+        self.logger.debug(f"Collected {len(resources)} Glue resources in {self.region}")
+        return resources
+
+    def _collect_databases(self, client, account_id: str) -> List[Resource]:
+        """Collect Glue databases and their tables."""
+        resources = []
+
+        try:
+            paginator = client.get_paginator("get_databases")
+            for page in paginator.paginate():
+                for db in page.get("DatabaseList", []):
+                    db_name = db.get("Name")
+                    db_arn = f"arn:aws:glue:{self.region}:{account_id}:database/{db_name}"
+
+                    resource = Resource(
+                        arn=db_arn,
+                        resource_type="AWS::Glue::Database",
+                        name=db_name,
+                        region=self.region,
+                        tags={},  # Glue databases don't support tags directly
+                        config_hash=compute_config_hash(db),
+                        created_at=db.get("CreateTime"),
+                        raw_config=db,
+                    )
+                    resources.append(resource)
+
+                    # Collect tables for this database
+                    resources.extend(self._collect_tables(client, account_id, db_name))
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Glue databases in {self.region}: {e}")
+
+        return resources
+
+    def _collect_tables(self, client, account_id: str, database_name: str) -> List[Resource]:
+        """Collect tables for a specific database."""
+        resources = []
+
+        try:
+            paginator = client.get_paginator("get_tables")
+            for page in paginator.paginate(DatabaseName=database_name):
+                for table in page.get("TableList", []):
+                    table_name = table.get("Name")
+                    table_arn = f"arn:aws:glue:{self.region}:{account_id}:table/{database_name}/{table_name}"
+
+                    resource = Resource(
+                        arn=table_arn,
+                        resource_type="AWS::Glue::Table",
+                        name=f"{database_name}/{table_name}",
+                        region=self.region,
+                        tags={},  # Glue tables don't support tags directly
+                        config_hash=compute_config_hash(table),
+                        created_at=table.get("CreateTime"),
+                        raw_config=table,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.debug(f"Error collecting tables for database {database_name}: {e}")
+
+        return resources
+
+    def _collect_crawlers(self, client, account_id: str) -> List[Resource]:
+        """Collect Glue crawlers."""
+        resources = []
+
+        try:
+            paginator = client.get_paginator("get_crawlers")
+            for page in paginator.paginate():
+                for crawler in page.get("Crawlers", []):
+                    crawler_name = crawler.get("Name")
+                    crawler_arn = f"arn:aws:glue:{self.region}:{account_id}:crawler/{crawler_name}"
+
+                    # Get tags for crawler
+                    tags = {}
+                    try:
+                        tag_response = client.get_tags(ResourceArn=crawler_arn)
+                        tags = tag_response.get("Tags", {})
+                    except Exception as e:
+                        self.logger.debug(f"Could not get tags for crawler {crawler_name}: {e}")
+
+                    resource = Resource(
+                        arn=crawler_arn,
+                        resource_type="AWS::Glue::Crawler",
+                        name=crawler_name,
+                        region=self.region,
+                        tags=tags,
+                        config_hash=compute_config_hash(crawler),
+                        created_at=crawler.get("CreationTime"),
+                        raw_config=crawler,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Glue crawlers in {self.region}: {e}")
+
+        return resources
+
+    def _collect_jobs(self, client, account_id: str) -> List[Resource]:
+        """Collect Glue jobs."""
+        resources = []
+
+        try:
+            paginator = client.get_paginator("get_jobs")
+            for page in paginator.paginate():
+                for job in page.get("Jobs", []):
+                    job_name = job.get("Name")
+                    job_arn = f"arn:aws:glue:{self.region}:{account_id}:job/{job_name}"
+
+                    # Get tags for job
+                    tags = {}
+                    try:
+                        tag_response = client.get_tags(ResourceArn=job_arn)
+                        tags = tag_response.get("Tags", {})
+                    except Exception as e:
+                        self.logger.debug(f"Could not get tags for job {job_name}: {e}")
+
+                    resource = Resource(
+                        arn=job_arn,
+                        resource_type="AWS::Glue::Job",
+                        name=job_name,
+                        region=self.region,
+                        tags=tags,
+                        config_hash=compute_config_hash(job),
+                        created_at=job.get("CreatedOn"),
+                        raw_config=job,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Glue jobs in {self.region}: {e}")
+
+        return resources
+
+    def _collect_connections(self, client, account_id: str) -> List[Resource]:
+        """Collect Glue connections."""
+        resources = []
+
+        try:
+            paginator = client.get_paginator("get_connections")
+            for page in paginator.paginate():
+                for conn in page.get("ConnectionList", []):
+                    conn_name = conn.get("Name")
+                    conn_arn = f"arn:aws:glue:{self.region}:{account_id}:connection/{conn_name}"
+
+                    resource = Resource(
+                        arn=conn_arn,
+                        resource_type="AWS::Glue::Connection",
+                        name=conn_name,
+                        region=self.region,
+                        tags={},  # Connections don't support tags
+                        config_hash=compute_config_hash(conn),
+                        created_at=conn.get("CreationTime"),
+                        raw_config=conn,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Glue connections in {self.region}: {e}")
+
+        return resources

src/snapshot/resource_collectors/iam.py

@@ -0,0 +1,188 @@
+"""IAM resource collector."""
+
+from typing import List
+
+from ...models.resource import Resource
+from ...utils.hash import compute_config_hash
+from .base import BaseResourceCollector
+
+
+class IAMCollector(BaseResourceCollector):
+    """Collector for AWS IAM resources (roles, users, groups, policies)."""
+
+    @property
+    def service_name(self) -> str:
+        return "iam"
+
+    @property
+    def is_global_service(self) -> bool:
+        return True
+
+    def collect(self) -> List[Resource]:
+        """Collect IAM resources.
+
+        Returns:
+            List of IAM resources (roles, users, groups, policies)
+        """
+        resources = []
+        account_id = self._get_account_id()
+
+        # Collect roles
+        resources.extend(self._collect_roles(account_id))
+
+        # Collect users
+        resources.extend(self._collect_users(account_id))
+
+        # Collect groups
+        resources.extend(self._collect_groups(account_id))
+
+        # Collect policies (customer-managed only)
+        resources.extend(self._collect_policies(account_id))
+
+        self.logger.debug(f"Collected {len(resources)} IAM resources")
+        return resources
+
+    def _collect_roles(self, account_id: str) -> List[Resource]:
+        """Collect IAM roles."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_roles")
+            for page in paginator.paginate():
+                for role in page["Roles"]:
+                    # Build ARN
+                    arn = role["Arn"]
+
+                    # Extract tags
+                    tags = {}
+                    try:
+                        tag_response = client.list_role_tags(RoleName=role["RoleName"])
+                        tags = {tag["Key"]: tag["Value"] for tag in tag_response.get("Tags", [])}
+                    except Exception as e:
+                        self.logger.debug(f"Could not get tags for role {role['RoleName']}: {e}")
+
+                    # Create resource
+                    resource = Resource(
+                        arn=arn,
+                        resource_type="AWS::IAM::Role",
+                        name=role["RoleName"],
+                        region="global",
+                        tags=tags,
+                        config_hash=compute_config_hash(role),
+                        created_at=role.get("CreateDate"),
+                        raw_config=role,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting IAM roles: {e}")
+
+        return resources
+
+    def _collect_users(self, account_id: str) -> List[Resource]:
+        """Collect IAM users."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_users")
+            for page in paginator.paginate():
+                for user in page["Users"]:
+                    # Build ARN
+                    arn = user["Arn"]
+
+                    # Extract tags
+                    tags = {}
+                    try:
+                        tag_response = client.list_user_tags(UserName=user["UserName"])
+                        tags = {tag["Key"]: tag["Value"] for tag in tag_response.get("Tags", [])}
+                    except Exception as e:
+                        self.logger.debug(f"Could not get tags for user {user['UserName']}: {e}")
+
+                    # Create resource
+                    resource = Resource(
+                        arn=arn,
+                        resource_type="AWS::IAM::User",
+                        name=user["UserName"],
+                        region="global",
+                        tags=tags,
+                        config_hash=compute_config_hash(user),
+                        created_at=user.get("CreateDate"),
+                        raw_config=user,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting IAM users: {e}")
+
+        return resources
+
+    def _collect_groups(self, account_id: str) -> List[Resource]:
+        """Collect IAM groups."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_groups")
+            for page in paginator.paginate():
+                for group in page["Groups"]:
+                    # Build ARN
+                    arn = group["Arn"]
+
+                    # Create resource (groups don't support tags)
+                    resource = Resource(
+                        arn=arn,
+                        resource_type="AWS::IAM::Group",
+                        name=group["GroupName"],
+                        region="global",
+                        tags={},
+                        config_hash=compute_config_hash(group),
+                        created_at=group.get("CreateDate"),
+                        raw_config=group,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting IAM groups: {e}")
+
+        return resources
+
+    def _collect_policies(self, account_id: str) -> List[Resource]:
+        """Collect customer-managed IAM policies."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_policies")
+            # Only get customer-managed policies (not AWS-managed)
+            for page in paginator.paginate(Scope="Local"):
+                for policy in page["Policies"]:
+                    # Build ARN
+                    arn = policy["Arn"]
+
+                    # Extract tags
+                    tags = {}
+                    try:
+                        tag_response = client.list_policy_tags(PolicyArn=arn)
+                        tags = {tag["Key"]: tag["Value"] for tag in tag_response.get("Tags", [])}
+                    except Exception as e:
+                        self.logger.debug(f"Could not get tags for policy {policy['PolicyName']}: {e}")
+
+                    # Create resource
+                    resource = Resource(
+                        arn=arn,
+                        resource_type="AWS::IAM::Policy",
+                        name=policy["PolicyName"],
+                        region="global",
+                        tags=tags,
+                        config_hash=compute_config_hash(policy),
+                        created_at=policy.get("CreateDate"),
+                        raw_config=policy,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting IAM policies: {e}")
+
+        return resources

src/snapshot/resource_collectors/kms.py

@@ -0,0 +1,111 @@
+"""KMS resource collector."""
+
+from typing import List
+
+from ...models.resource import Resource
+from ...utils.hash import compute_config_hash
+from .base import BaseResourceCollector
+
+
+class KMSCollector(BaseResourceCollector):
+    """Collector for AWS KMS (Key Management Service) resources."""
+
+    @property
+    def service_name(self) -> str:
+        return "kms"
+
+    def collect(self) -> List[Resource]:
+        """Collect KMS keys.
+
+        Collects customer-managed KMS keys (not AWS-managed keys).
+
+        Returns:
+            List of KMS key resources
+        """
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_keys")
+            for page in paginator.paginate():
+                for key_item in page.get("Keys", []):
+                    key_id = key_item["KeyId"]
+                    key_arn = key_item["KeyArn"]
+
+                    try:
+                        # Get key metadata
+                        key_metadata = client.describe_key(KeyId=key_id)["KeyMetadata"]
+
+                        # Skip AWS-managed keys (we only want customer-managed keys)
+                        if key_metadata.get("KeyManager") == "AWS":
+                            continue
+
+                        # Skip keys that are pending deletion
+                        if key_metadata.get("KeyState") == "PendingDeletion":
+                            self.logger.debug(f"Skipping key {key_id} - pending deletion")
+                            continue
+
+                        key_alias = None
+
+                        # Get key aliases
+                        try:
+                            aliases_response = client.list_aliases(KeyId=key_id)
+                            aliases = aliases_response.get("Aliases", [])
+                            if aliases:
+                                # Use first alias as the name
+                                key_alias = aliases[0]["AliasName"]
+                        except Exception as e:
+                            self.logger.debug(f"Could not get aliases for key {key_id}: {e}")
+
+                        # Get tags
+                        tags = {}
+                        try:
+                            tag_response = client.list_resource_tags(KeyId=key_id)
+                            for tag in tag_response.get("Tags", []):
+                                tags[tag["TagKey"]] = tag["TagValue"]
+                        except Exception as e:
+                            self.logger.debug(f"Could not get tags for key {key_id}: {e}")
+
+                        # Get key rotation status
+                        rotation_enabled = False
+                        try:
+                            rotation_response = client.get_key_rotation_status(KeyId=key_id)
+                            rotation_enabled = rotation_response.get("KeyRotationEnabled", False)
+                        except Exception as e:
+                            self.logger.debug(f"Could not get rotation status for key {key_id}: {e}")
+
+                        # Build config for hash
+                        config = {
+                            **key_metadata,
+                            "RotationEnabled": rotation_enabled,
+                            "Aliases": [key_alias] if key_alias else [],
+                        }
+
+                        # Extract creation date
+                        created_at = key_metadata.get("CreationDate")
+
+                        # Use alias as name if available, otherwise use key ID
+                        name = key_alias if key_alias else key_id
+
+                        # Create resource
+                        resource = Resource(
+                            arn=key_arn,
+                            resource_type="AWS::KMS::Key",
+                            name=name,
+                            region=self.region,
+                            tags=tags,
+                            config_hash=compute_config_hash(config),
+                            created_at=created_at,
+                            raw_config=config,
+                        )
+                        resources.append(resource)
+
+                    except Exception as e:
+                        self.logger.debug(f"Error processing key {key_id}: {e}")
+                        continue
+
+        except Exception as e:
+            self.logger.error(f"Error collecting KMS keys in {self.region}: {e}")
+
+        self.logger.debug(f"Collected {len(resources)} KMS keys in {self.region}")
+        return resources

src/snapshot/resource_collectors/lambda_func.py

@@ -0,0 +1,139 @@
+"""Lambda resource collector."""
+
+from datetime import datetime
+from typing import List, Optional
+
+from ...models.resource import Resource
+from ...utils.hash import compute_config_hash
+from .base import BaseResourceCollector
+
+
+def _parse_lambda_timestamp(timestamp_str: Optional[str]) -> Optional[datetime]:
+    """Parse Lambda's ISO-8601 timestamp format.
+
+    Lambda returns timestamps like: 2024-01-15T10:30:00.000+0000
+
+    Args:
+        timestamp_str: ISO-8601 formatted timestamp string
+
+    Returns:
+        Parsed datetime or None if parsing fails
+    """
+    if not timestamp_str:
+        return None
+    try:
+        # Lambda format: 2024-01-15T10:30:00.000+0000
+        # Python's fromisoformat doesn't handle +0000 format, need to normalize
+        normalized = timestamp_str.replace("+0000", "+00:00").replace("-0000", "+00:00")
+        return datetime.fromisoformat(normalized)
+    except (ValueError, AttributeError):
+        return None
+
+
+class LambdaCollector(BaseResourceCollector):
+    """Collector for AWS Lambda functions and layers."""
+
+    @property
+    def service_name(self) -> str:
+        return "lambda"
+
+    def collect(self) -> List[Resource]:
+        """Collect Lambda resources.
+
+        Returns:
+            List of Lambda functions and layers
+        """
+        resources = []
+        account_id = self._get_account_id()
+
+        # Collect functions
+        resources.extend(self._collect_functions(account_id))
+
+        # Collect layers
+        resources.extend(self._collect_layers(account_id))
+
+        self.logger.debug(f"Collected {len(resources)} Lambda resources in {self.region}")
+        return resources
+
+    def _collect_functions(self, account_id: str) -> List[Resource]:
+        """Collect Lambda functions."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_functions")
+            for page in paginator.paginate():
+                for function in page["Functions"]:
+                    function_name = function["FunctionName"]
+                    function_arn = function["FunctionArn"]
+
+                    # Get full function configuration (includes tags)
+                    try:
+                        full_config = client.get_function(FunctionName=function_name)
+                        tags = full_config.get("Tags", {})
+                        config_data = full_config.get("Configuration", function)
+                    except Exception as e:
+                        self.logger.debug(f"Could not get full config for {function_name}: {e}")
+                        tags = {}
+                        config_data = function
+
+                    # Parse LastModified timestamp (set on creation and updates)
+                    last_modified = _parse_lambda_timestamp(config_data.get("LastModified"))
+
+                    # Create resource
+                    resource = Resource(
+                        arn=function_arn,
+                        resource_type="AWS::Lambda::Function",
+                        name=function_name,
+                        region=self.region,
+                        tags=tags,
+                        config_hash=compute_config_hash(config_data),
+                        created_at=last_modified,
+                        raw_config=config_data,
+                    )
+                    resources.append(resource)
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Lambda functions in {self.region}: {e}")
+
+        return resources
+
+    def _collect_layers(self, account_id: str) -> List[Resource]:
+        """Collect Lambda layers."""
+        resources = []
+        client = self._create_client()
+
+        try:
+            paginator = client.get_paginator("list_layers")
+            for page in paginator.paginate():
+                for layer in page["Layers"]:
+                    layer_name = layer["LayerName"]
+                    layer_arn = layer["LayerArn"]
+
+                    # Get latest version info
+                    try:
+                        latest_version = layer.get("LatestMatchingVersion", {})
+                        layer_version_arn = latest_version.get("LayerVersionArn", layer_arn)
+                        # Parse CreatedDate timestamp (same format as function LastModified)
+                        created_at = _parse_lambda_timestamp(latest_version.get("CreatedDate"))
+
+                        # Create resource
+                        resource = Resource(
+                            arn=layer_version_arn,
+                            resource_type="AWS::Lambda::LayerVersion",
+                            name=layer_name,
+                            region=self.region,
+                            tags={},  # Layers don't support tags
+                            config_hash=compute_config_hash(layer),
+                            created_at=created_at,
+                            raw_config=layer,
+                        )
+                        resources.append(resource)
+
+                    except Exception as e:
+                        self.logger.debug(f"Could not get layer version for {layer_name}: {e}")
+
+        except Exception as e:
+            self.logger.error(f"Error collecting Lambda layers in {self.region}: {e}")
+
+        return resources