nuvu-scan 2.0.2__py3-none-any.whl → 2.1.2__py3-none-any.whl

nuvu_scan/core/providers/aws/collectors/athena.py

@@ -1,10 +1,10 @@
 """
 Amazon Athena collector.
 
-Collects Athena workgroups and query history.
+Collects Athena workgroups and query history across all regions.
 """
 
-from datetime import datetime, timedelta
+from datetime import datetime, timezone
 from typing import Any
 
 import boto3
@@ -14,90 +14,121 @@ from nuvu_scan.core.base import Asset, NormalizedCategory
 
 
 class AthenaCollector:
-    """Collects Amazon Athena resources."""
+    """Collects Amazon Athena resources across all regions."""
 
     def __init__(self, session: boto3.Session, regions: list[str] | None = None):
         self.session = session
-        self.regions = regions or ["us-east-1"]
-        self.athena_client = session.client("athena", region_name="us-east-1")
+        self.regions = regions or []
 
     def collect(self) -> list[Asset]:
-        """Collect Athena workgroups."""
+        """Collect Athena workgroups from all regions."""
         import sys
 
         assets = []
 
-        try:
-            # List workgroups
-            print("  → Listing Athena workgroups...", file=sys.stderr)
-            response = self.athena_client.list_work_groups()
-
-            for wg_info in response.get("WorkGroups", []):
-                wg_name = wg_info["Name"]
-
-                try:
-                    # Get workgroup details
-                    wg_details = self.athena_client.get_work_group(WorkGroup=wg_name)
-                    wg_details.get("WorkGroup", {}).get("Configuration", {})
-
-                    # Get query statistics
-                    query_stats = self._get_query_stats(wg_name)
-
-                    risk_flags = []
-                    if query_stats.get("idle_days", 0) > 90:
-                        risk_flags.append("idle_workgroup")
-                    if (
-                        query_stats.get("failed_queries", 0)
-                        > query_stats.get("total_queries", 1) * 0.5
-                    ):
-                        risk_flags.append("high_failure_rate")
-
-                    assets.append(
-                        Asset(
-                            provider="aws",
-                            asset_type="athena_workgroup",
-                            normalized_category=NormalizedCategory.QUERY_ENGINE,
-                            service="Athena",
-                            region="us-east-1",
-                            arn=f"arn:aws:athena:us-east-1::workgroup/{wg_name}",
-                            name=wg_name,
-                            created_at=(
-                                wg_details.get("WorkGroup", {}).get("CreationTime", "").isoformat()
-                                if wg_details.get("WorkGroup", {}).get("CreationTime")
-                                else None
-                            ),
-                            last_activity_at=query_stats.get("last_query_time"),
-                            risk_flags=risk_flags,
-                            usage_metrics={
-                                **query_stats,
-                                "last_used": query_stats.get("last_query_time"),
-                                "days_since_last_use": query_stats.get("idle_days"),
-                            },
+        # If no regions specified, get all enabled regions
+        regions_to_check = self.regions if self.regions else self._get_all_regions()
+
+        print(
+            f"  → Checking {len(regions_to_check)} regions for Athena workgroups...",
+            file=sys.stderr,
+        )
+
+        for region in regions_to_check:
+            try:
+                athena_client = self.session.client("athena", region_name=region)
+                response = athena_client.list_work_groups()
+
+                for wg_info in response.get("WorkGroups", []):
+                    wg_name = wg_info["Name"]
+
+                    try:
+                        # Get workgroup details
+                        wg_details = athena_client.get_work_group(WorkGroup=wg_name)
+
+                        # Get query statistics
+                        query_stats = self._get_query_stats(athena_client, wg_name)
+
+                        risk_flags = []
+                        if query_stats.get("idle_days", 0) > 90:
+                            risk_flags.append("idle_workgroup")
+                        if (
+                            query_stats.get("failed_queries", 0)
+                            > query_stats.get("total_queries", 1) * 0.5
+                        ):
+                            risk_flags.append("high_failure_rate")
+
+                        assets.append(
+                            Asset(
+                                provider="aws",
+                                asset_type="athena_workgroup",
+                                normalized_category=NormalizedCategory.QUERY_ENGINE,
+                                service="Athena",
+                                region=region,
+                                arn=f"arn:aws:athena:{region}::workgroup/{wg_name}",
+                                name=wg_name,
+                                created_at=(
+                                    wg_details.get("WorkGroup", {})
+                                    .get("CreationTime", "")
+                                    .isoformat()
+                                    if wg_details.get("WorkGroup", {}).get("CreationTime")
+                                    else None
+                                ),
+                                last_activity_at=query_stats.get("last_query_time"),
+                                risk_flags=risk_flags,
+                                usage_metrics={
+                                    **query_stats,
+                                    "last_used": query_stats.get("last_query_time"),
+                                    "days_since_last_use": query_stats.get("idle_days"),
+                                },
+                            )
                         )
-                    )
-                except ClientError:
-                    continue
+                    except ClientError:
+                        continue
 
-        except ClientError as e:
-            print(f"Error collecting Athena resources: {e}")
+            except ClientError as e:
+                error_code = e.response.get("Error", {}).get("Code", "Unknown")
+                if error_code == "AccessDeniedException":
+                    print(
+                        f"  ⚠️  No permission to list Athena workgroups in {region}. "
+                        "Add 'athena:ListWorkGroups' to IAM policy.",
+                        file=sys.stderr,
+                    )
+                # Skip other errors silently (region not enabled, etc.)
 
+        if assets:
+            print(f"  → Found {len(assets)} Athena workgroups", file=sys.stderr)
         return assets
 
-    def _get_query_stats(self, workgroup_name: str) -> dict[str, Any]:
+    def _get_all_regions(self) -> list[str]:
+        """Get all enabled AWS regions."""
+        try:
+            ec2 = self.session.client("ec2", region_name="us-east-1")
+            response = ec2.describe_regions(AllRegions=False)
+            return [r["RegionName"] for r in response.get("Regions", [])]
+        except ClientError:
+            # Fallback to common regions
+            return ["us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"]
+
+    def _get_query_stats(self, athena_client, workgroup_name: str) -> dict[str, Any]:
         """Get query statistics for a workgroup."""
         stats = {"total_queries": 0, "failed_queries": 0, "last_query_time": None, "idle_days": 0}
 
         try:
-            # List recent queries
-            paginator = self.athena_client.get_paginator("list_query_executions")
-            datetime.utcnow() - timedelta(days=90)
+            # List recent queries (limit to avoid long scan times)
+            paginator = athena_client.get_paginator("list_query_executions")
 
-            for page in paginator.paginate(WorkGroup=workgroup_name):
+            query_count = 0
+            for page in paginator.paginate(
+                WorkGroup=workgroup_name, PaginationConfig={"MaxItems": 100}
+            ):
                 for query_id in page.get("QueryExecutionIds", []):
+                    query_count += 1
+                    if query_count > 50:  # Limit for performance
+                        break
+
                     try:
-                        query_info = self.athena_client.get_query_execution(
-                            QueryExecutionId=query_id
-                        )
+                        query_info = athena_client.get_query_execution(QueryExecutionId=query_id)
                         execution = query_info.get("QueryExecution", {})
                         status = execution.get("Status", {})
 
@@ -107,7 +138,7 @@ class AthenaCollector:
                             stats["failed_queries"] += 1
 
                         # Get last query time
-                        completion_time = execution.get("Status", {}).get("CompletionDateTime")
+                        completion_time = status.get("CompletionDateTime")
                         if completion_time:
                             if (
                                 not stats["last_query_time"]
@@ -117,10 +148,14 @@ class AthenaCollector:
                     except ClientError:
                         continue
 
+                if query_count > 50:
+                    break
+
             # Calculate idle days
             if stats["last_query_time"]:
                 last_query = datetime.fromisoformat(stats["last_query_time"].replace("Z", "+00:00"))
-                stats["idle_days"] = (datetime.utcnow() - last_query.replace(tzinfo=None)).days
+                now = datetime.now(timezone.utc)
+                stats["idle_days"] = (now - last_query).days
             else:
                 stats["idle_days"] = 999  # Never used
 

nuvu_scan/core/providers/aws/collectors/glue.py

@@ -14,48 +14,79 @@ from nuvu_scan.core.base import Asset, NormalizedCategory
 
 
 class GlueCollector:
-    """Collects AWS Glue Data Catalog resources."""
+    """Collects AWS Glue Data Catalog resources across all regions."""
 
     def __init__(self, session: boto3.Session, regions: list[str] | None = None):
         self.session = session
-        self.regions = regions or ["us-east-1"]  # Glue is regional but catalog is global
-        self.region = self.regions[0] if self.regions else "us-east-1"
-        self.glue_client = session.client("glue", region_name=self.region)
+        self.regions = regions or []
+        # These will be set per-region during collection
+        self.glue_client = None
+        self.region = None
         # Cache crawler run times to associate with tables
         self._crawler_last_runs: dict[str, datetime | None] = {}
         self._db_to_crawler: dict[str, str] = {}
 
+    def _get_all_regions(self) -> list[str]:
+        """Get all enabled AWS regions."""
+        try:
+            ec2 = self.session.client("ec2", region_name="us-east-1")
+            response = ec2.describe_regions(AllRegions=False)
+            return [r["RegionName"] for r in response.get("Regions", [])]
+        except ClientError:
+            # Fallback to common regions
+            return ["us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"]
+
     def collect(self) -> list[Asset]:
-        """Collect all Glue resources: databases, tables, crawlers, jobs, connections."""
+        """Collect all Glue resources across all regions."""
         import sys
 
-        assets = []
+        all_assets = []
 
-        # First, collect crawlers to build database-to-crawler mapping
-        print("  → Collecting Glue crawlers...", file=sys.stderr)
-        crawler_assets = self._collect_crawlers()
-        assets.extend(crawler_assets)
-        print(f"  → Found {len(crawler_assets)} crawlers", file=sys.stderr)
-
-        # Collect ETL jobs
-        print("  → Collecting Glue ETL jobs...", file=sys.stderr)
-        job_assets = self._collect_jobs()
-        assets.extend(job_assets)
-        print(f"  → Found {len(job_assets)} jobs", file=sys.stderr)
-
-        # Collect connections
-        print("  → Collecting Glue connections...", file=sys.stderr)
-        conn_assets = self._collect_connections()
-        assets.extend(conn_assets)
-        print(f"  → Found {len(conn_assets)} connections", file=sys.stderr)
-
-        # Collect databases and tables (using crawler info for last_activity)
-        print("  → Collecting Glue databases and tables...", file=sys.stderr)
-        db_assets = self._collect_databases_and_tables()
-        assets.extend(db_assets)
-        print(f"  → Found {len(db_assets)} databases/tables", file=sys.stderr)
+        # If no regions specified, get all enabled regions
+        regions_to_check = self.regions if self.regions else self._get_all_regions()
 
-        return assets
+        print(
+            f"  → Checking {len(regions_to_check)} regions for Glue resources...", file=sys.stderr
+        )
+
+        total_crawlers = 0
+        total_jobs = 0
+        total_connections = 0
+        total_dbs = 0
+
+        for region in regions_to_check:
+            # Set up client for this region
+            self.region = region
+            self.glue_client = self.session.client("glue", region_name=region)
+            self._crawler_last_runs = {}
+            self._db_to_crawler = {}
+
+            # Collect crawlers
+            crawler_assets = self._collect_crawlers()
+            all_assets.extend(crawler_assets)
+            total_crawlers += len(crawler_assets)
+
+            # Collect ETL jobs
+            job_assets = self._collect_jobs()
+            all_assets.extend(job_assets)
+            total_jobs += len(job_assets)
+
+            # Collect connections
+            conn_assets = self._collect_connections()
+            all_assets.extend(conn_assets)
+            total_connections += len(conn_assets)
+
+            # Collect databases and tables
+            db_assets = self._collect_databases_and_tables()
+            all_assets.extend(db_assets)
+            total_dbs += len(db_assets)
+
+        print(f"  → Found {total_crawlers} crawlers", file=sys.stderr)
+        print(f"  → Found {total_jobs} jobs", file=sys.stderr)
+        print(f"  → Found {total_connections} connections", file=sys.stderr)
+        print(f"  → Found {total_dbs} databases/tables", file=sys.stderr)
+
+        return all_assets
 
     def _collect_crawlers(self) -> list[Asset]:
         """Collect Glue Crawlers with detailed status."""
@@ -152,7 +183,17 @@ class GlueCollector:
                     )
 
         except ClientError as e:
-            print(f"Error collecting Glue crawlers: {e}")
+            import sys
+
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDeniedException":
+                print(
+                    "  ⚠️  No permission to list Glue crawlers. "
+                    "Add 'glue:GetCrawlers' to IAM policy.",
+                    file=sys.stderr,
+                )
+            else:
+                print(f"  ⚠️  Error collecting Glue crawlers: {e}", file=sys.stderr)
 
         return assets
 
@@ -248,7 +289,16 @@ class GlueCollector:
                     )
 
         except ClientError as e:
-            print(f"Error collecting Glue jobs: {e}")
+            import sys
+
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDeniedException":
+                print(
+                    "  ⚠️  No permission to list Glue jobs. Add 'glue:GetJobs' to IAM policy.",
+                    file=sys.stderr,
+                )
+            else:
+                print(f"  ⚠️  Error collecting Glue jobs: {e}", file=sys.stderr)
 
         return assets
 
@@ -307,7 +357,17 @@ class GlueCollector:
                 )
 
         except ClientError as e:
-            print(f"Error collecting Glue connections: {e}")
+            import sys
+
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDeniedException":
+                print(
+                    "  ⚠️  No permission to list Glue connections. "
+                    "Add 'glue:GetConnections' to IAM policy.",
+                    file=sys.stderr,
+                )
+            else:
+                print(f"  ⚠️  Error collecting Glue connections: {e}", file=sys.stderr)
 
         return assets
 
@@ -452,7 +512,17 @@ class GlueCollector:
                         pass
 
         except ClientError as e:
-            print(f"Error collecting Glue resources: {e}")
+            import sys
+
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            if error_code == "AccessDeniedException":
+                print(
+                    "  ⚠️  No permission to list Glue databases/tables. "
+                    "Add 'glue:GetDatabases' and 'glue:GetTables' to IAM policy.",
+                    file=sys.stderr,
+                )
+            else:
+                print(f"  ⚠️  Error collecting Glue resources: {e}", file=sys.stderr)
 
         return assets
 

nuvu_scan/core/providers/aws/collectors/mwaa.py

@@ -91,8 +91,8 @@ class MWAACollector:
                                 environment_class, min_workers, max_workers
                             )
 
-                            # Check for risk flags
-                            risk_flags = self._check_risks(environment, tags)
+                            # Check for risk flags (pass ownership for proper no_owner detection)
+                            risk_flags = self._check_risks(environment, tags, ownership)
 
                             # Create asset
                             asset = Asset(
@@ -251,7 +251,9 @@ class MWAACollector:
 
         return base_monthly + worker_monthly
 
-    def _check_risks(self, environment: dict, tags: dict[str, str]) -> list[str]:
+    def _check_risks(
+        self, environment: dict, tags: dict[str, str], ownership: dict | None = None
+    ) -> list[str]:
         """Check for risk flags in MWAA environment."""
         risks = []
 
@@ -260,8 +262,11 @@ class MWAACollector:
         if status in ["CREATING", "DELETING", "UPDATING"]:
             risks.append("environment_in_transition")
 
-        # Check for missing owner
-        if not tags.get("owner") and not tags.get("Owner"):
+        # Check for missing owner - only flag if we couldn't infer owner at all
+        if ownership:
+            if not ownership.get("owner") and ownership.get("confidence") == "unknown":
+                risks.append("no_owner")
+        elif not tags.get("owner") and not tags.get("Owner"):
             risks.append("no_owner")
 
         # Check for public access (if webserver is publicly accessible)