iam-policy-validator 1.13.1__py3-none-any.whl → 1.14.1__py3-none-any.whl

iam_validator/commands/validate.py

@@ -64,6 +64,10 @@ Examples:
 
   # Only GitHub Actions job summary
   iam-validator validate --path ./policies/ --github-summary
+
+  # CI mode: show enhanced output in logs, save JSON to file
+  iam-validator validate --path ./policies/ --ci --github-review
+  iam-validator validate --path ./policies/ --ci --ci-output results.json
         """
 
     def add_arguments(self, parser: argparse.ArgumentParser) -> None:
@@ -198,6 +202,33 @@ Examples:
             help="Show Issue Severity Breakdown section in enhanced format output",
         )
 
+        parser.add_argument(
+            "--allow-owner-ignore",
+            action="store_true",
+            default=True,
+            help="Allow CODEOWNERS to ignore findings by replying 'ignore' to review comments (default: enabled)",
+        )
+
+        parser.add_argument(
+            "--no-owner-ignore",
+            action="store_true",
+            help="Disable CODEOWNERS ignore feature",
+        )
+
+        parser.add_argument(
+            "--ci",
+            action="store_true",
+            help="CI mode: print enhanced console output for visibility in job logs, "
+            "and write JSON report to file (use --ci-output to specify filename, "
+            "defaults to 'validation-report.json').",
+        )
+
+        parser.add_argument(
+            "--ci-output",
+            default="validation-report.json",
+            help="Output file for JSON report in CI mode (default: validation-report.json)",
+        )
+
     async def execute(self, args: argparse.Namespace) -> int:
         """Execute the validate command."""
         # Check if streaming mode is enabled
@@ -266,8 +297,15 @@ Examples:
         generator = ReportGenerator()
         report = generator.generate_report(results, parsing_errors=loader.parsing_errors)
 
-        # Output results
-        if args.format is None:
+        # Handle --ci flag: show enhanced output in console, write JSON to file
+        ci_mode = getattr(args, "ci", False)
+        if ci_mode:
+            # CI mode: enhanced output to console, JSON to file
+            self._print_ci_console_output(report, generator)
+            ci_output_file = getattr(args, "ci_output", "validation-report.json")
+            generator.save_json_report(report, ci_output_file)
+            logging.info(f"Saved JSON report to {ci_output_file}")
+        elif args.format is None:
             # Default: use classic console output (direct Rich printing)
             generator.print_console_report(report)
         elif args.format == "json":
@@ -302,16 +340,26 @@ Examples:
             from iam_validator.core.config.config_loader import ConfigLoader
             from iam_validator.core.pr_commenter import PRCommenter
 
-            # Load config to get fail_on_severity and severity_labels settings
+            # Load config to get fail_on_severity, severity_labels, and ignore settings
             config = ConfigLoader.load_config(config_path)
             fail_on_severities = config.get_setting("fail_on_severity", ["error", "critical"])
             severity_labels = config.get_setting("severity_labels", {})
 
+            # Get ignore settings from config, but CLI flag can override
+            ignore_settings = config.get_setting("ignore_settings", {})
+            enable_ignore = ignore_settings.get("enabled", True)
+            # CLI --no-owner-ignore takes precedence
+            if getattr(args, "no_owner_ignore", False):
+                enable_ignore = False
+            allowed_users = ignore_settings.get("allowed_users", [])
+
             async with GitHubIntegration() as github:
                 commenter = PRCommenter(
                     github,
                     fail_on_severities=fail_on_severities,
                     severity_labels=severity_labels,
+                    enable_codeowners_ignore=enable_ignore,
+                    allowed_ignore_users=allowed_users,
                 )
                 success = await commenter.post_findings_to_pr(
                     report,
@@ -348,10 +396,8 @@ Examples:
 
         all_results = []
         total_processed = 0
-
-        # Clean up old review comments at the start (before posting any new ones)
-        if getattr(args, "github_review", False):
-            await self._cleanup_old_comments(args)
+        # Track all validated files across the streaming session for final cleanup
+        all_validated_files: set[str] = set()
 
         logging.info(f"Starting streaming validation from {len(args.paths)} path(s)")
 
@@ -374,6 +420,11 @@ Examples:
                 result = results[0]
                 all_results.append(result)
 
+                # Track validated file (convert to relative path for cleanup)
+                relative_path = self._make_relative_path(file_path)
+                if relative_path:
+                    all_validated_files.add(relative_path)
+
                 # Print immediate feedback for this file
                 if args.format == "console":
                     if result.is_valid:
@@ -383,6 +434,8 @@ Examples:
                         # Note: validation_success tracks overall status
 
                 # Post to GitHub immediately for this file (progressive PR comments)
+                # skip_cleanup=True because we process files one at a time and don't want
+                # to delete comments from files processed earlier. Cleanup runs at the end.
                 if getattr(args, "github_review", False):
                     await self._post_file_review(result, args)
 
@@ -392,11 +445,23 @@ Examples:
 
         logging.info(f"\nCompleted validation of {total_processed} policies")
 
+        # Run final cleanup after all files are processed
+        # This uses the full report to know all current findings and deletes stale comments
+        if getattr(args, "github_review", False):
+            await self._run_final_review_cleanup(args, all_results, all_validated_files)
+
         # Generate final summary report
         report = generator.generate_report(all_results)
 
-        # Output final results
-        if args.format == "console":
+        # Handle --ci flag: show enhanced output in console, write JSON to file
+        ci_mode = getattr(args, "ci", False)
+        if ci_mode:
+            # CI mode: enhanced output to console, JSON to file
+            self._print_ci_console_output(report, generator)
+            ci_output_file = getattr(args, "ci_output", "validation-report.json")
+            generator.save_json_report(report, ci_output_file)
+            logging.info(f"Saved JSON report to {ci_output_file}")
+        elif args.format == "console":
             # Classic console output (direct Rich printing from report.py)
             generator.print_console_report(report)
         elif args.format == "json":
@@ -431,16 +496,26 @@ Examples:
             from iam_validator.core.config.config_loader import ConfigLoader
             from iam_validator.core.pr_commenter import PRCommenter
 
-            # Load config to get fail_on_severity and severity_labels settings
+            # Load config to get fail_on_severity, severity_labels, and ignore settings
             config = ConfigLoader.load_config(config_path)
             fail_on_severities = config.get_setting("fail_on_severity", ["error", "critical"])
             severity_labels = config.get_setting("severity_labels", {})
 
+            # Get ignore settings from config, but CLI flag can override
+            ignore_settings = config.get_setting("ignore_settings", {})
+            enable_ignore = ignore_settings.get("enabled", True)
+            # CLI --no-owner-ignore takes precedence
+            if getattr(args, "no_owner_ignore", False):
+                enable_ignore = False
+            allowed_users = ignore_settings.get("allowed_users", [])
+
             async with GitHubIntegration() as github:
                 commenter = PRCommenter(
                     github,
                     fail_on_severities=fail_on_severities,
                     severity_labels=severity_labels,
+                    enable_codeowners_ignore=enable_ignore,
+                    allowed_ignore_users=allowed_users,
                 )
                 success = await commenter.post_findings_to_pr(
                     report,
@@ -487,24 +562,34 @@ Examples:
                 if not github.is_configured():
                     return
 
-                # Load config to get fail_on_severity setting
+                # Load config to get fail_on_severity and ignore settings
                 config_path = getattr(args, "config", None)
                 config = ConfigLoader.load_config(config_path)
                 fail_on_severities = config.get_setting("fail_on_severity", ["error", "critical"])
 
+                # Get ignore settings from config, but CLI flag can override
+                ignore_settings = config.get_setting("ignore_settings", {})
+                enable_ignore = ignore_settings.get("enabled", True)
+                # CLI --no-owner-ignore takes precedence
+                if getattr(args, "no_owner_ignore", False):
+                    enable_ignore = False
+                allowed_users = ignore_settings.get("allowed_users", [])
+
                 # In streaming mode, don't cleanup comments (we want to keep earlier files)
                 # Cleanup will happen once at the end
                 commenter = PRCommenter(
                     github,
                     cleanup_old_comments=False,
                     fail_on_severities=fail_on_severities,
+                    enable_codeowners_ignore=enable_ignore,
+                    allowed_ignore_users=allowed_users,
                 )
 
                 # Create a mini-report for just this file
                 generator = ReportGenerator()
                 mini_report = generator.generate_report([result])
 
-                # Post line-specific comments
+                # Post line-specific comments (skip cleanup - runs at end of streaming)
                 await commenter.post_findings_to_pr(
                     mini_report,
                     create_review=True,
@@ -513,6 +598,109 @@ Examples:
         except Exception as e:
             logging.warning(f"Failed to post review for {result.policy_file}: {e}")
 
+    def _make_relative_path(self, file_path: str) -> str | None:
+        """Convert absolute path to relative path for GitHub.
+
+        Args:
+            file_path: Absolute or relative path to file
+
+        Returns:
+            Relative path from repository root, or None if cannot be determined
+        """
+        from pathlib import Path
+
+        # If already relative, use as-is
+        if not os.path.isabs(file_path):
+            return file_path
+
+        # Try to get workspace path from environment
+        workspace = os.getenv("GITHUB_WORKSPACE")
+        if workspace:
+            try:
+                abs_file_path = Path(file_path).resolve()
+                workspace_path = Path(workspace).resolve()
+
+                if abs_file_path.is_relative_to(workspace_path):
+                    relative = abs_file_path.relative_to(workspace_path)
+                    return str(relative).replace("\\", "/")
+            except (ValueError, OSError) as exc:
+                logging.debug(f"Could not make path relative to GitHub workspace: {exc}")
+
+        # Fallback: try current working directory
+        try:
+            cwd = Path.cwd()
+            abs_file_path = Path(file_path).resolve()
+            if abs_file_path.is_relative_to(cwd):
+                relative = abs_file_path.relative_to(cwd)
+                return str(relative).replace("\\", "/")
+        except (ValueError, OSError) as exc:
+            logging.debug(f"Could not make path relative to cwd: {exc}")
+
+        return None
+
+    async def _run_final_review_cleanup(
+        self,
+        args: argparse.Namespace,
+        all_results: list,
+        all_validated_files: set[str],
+    ) -> None:
+        """Run final cleanup after all files are processed in streaming mode.
+
+        This deletes stale comments for findings that are no longer present,
+        using the complete set of validated files and current findings.
+
+        Args:
+            args: Command-line arguments
+            all_results: All validation results from the streaming session
+            all_validated_files: Set of all validated file paths (relative)
+        """
+        try:
+            from iam_validator.core.config.config_loader import ConfigLoader
+            from iam_validator.core.pr_commenter import PRCommenter
+
+            async with GitHubIntegration() as github:
+                if not github.is_configured():
+                    return
+
+                # Load config
+                config_path = getattr(args, "config", None)
+                config = ConfigLoader.load_config(config_path)
+                fail_on_severities = config.get_setting("fail_on_severity", ["error", "critical"])
+
+                # Get ignore settings
+                ignore_settings = config.get_setting("ignore_settings", {})
+                enable_ignore = ignore_settings.get("enabled", True)
+                if getattr(args, "no_owner_ignore", False):
+                    enable_ignore = False
+                allowed_users = ignore_settings.get("allowed_users", [])
+
+                # Create commenter WITH cleanup enabled for the final pass
+                commenter = PRCommenter(
+                    github,
+                    cleanup_old_comments=True,  # Enable cleanup for final pass
+                    fail_on_severities=fail_on_severities,
+                    enable_codeowners_ignore=enable_ignore,
+                    allowed_ignore_users=allowed_users,
+                )
+
+                # Create a full report with all results
+                generator = ReportGenerator()
+                full_report = generator.generate_report(all_results)
+
+                # Post with create_review=True to run the full update/create/delete logic
+                # but pass all_validated_files so cleanup knows the full scope
+                logging.info("Running final comment cleanup...")
+                await commenter.post_findings_to_pr(
+                    full_report,
+                    create_review=True,
+                    add_summary_comment=False,
+                    manage_labels=False,  # Labels are managed separately
+                    process_ignores=False,  # Already processed per-file
+                )
+
+        except Exception as e:
+            logging.warning(f"Failed to run final review cleanup: {e}")
+
     def _write_github_actions_summary(self, report: ValidationReport) -> None:
         """Write a high-level summary to GitHub Actions job summary.
 
@@ -609,3 +797,34 @@ Examples:
 
         except Exception as e:
             logging.warning(f"Failed to write GitHub Actions summary: {e}")
+
+    def _print_ci_console_output(
+        self, report: ValidationReport, generator: ReportGenerator
+    ) -> None:
+        """Print enhanced console output for CI visibility.
+
+        This shows validation results in the CI job logs in a human-readable format.
+        JSON output is written to a separate file (specified by --ci-output).
+
+        Args:
+            report: Validation report to print
+            generator: ReportGenerator instance
+        """
+        # Generate enhanced format output with summary and severity breakdown
+        try:
+            enhanced_output = generator.format_report(
+                report,
+                "enhanced",
+                show_summary=True,
+                show_severity_breakdown=True,
+            )
+            print(enhanced_output)
+
+        except Exception as e:
+            # Fallback to basic summary if enhanced format fails
+            logging.warning(f"Failed to generate enhanced output: {e}")
+            print("\nValidation Summary:")
+            print(f"  Total policies: {report.total_policies}")
+            print(f"  Valid: {report.valid_policies}")
+            print(f"  Invalid: {report.invalid_policies}")
+            print(f"  Total issues: {report.total_issues}\n")

iam_validator/core/aws_service/fetcher.py

@@ -100,6 +100,9 @@ class AWSServiceFetcher:
         "wafv2",
     ]
 
+    # Default concurrency limits
+    DEFAULT_MAX_CONCURRENT_REQUESTS = 10
+
     def __init__(
         self,
         timeout: float = constants.DEFAULT_HTTP_TIMEOUT_SECONDS,
@@ -112,6 +115,7 @@ class AWSServiceFetcher:
         prefetch_common: bool = True,
         cache_dir: Path | str | None = None,
         aws_services_dir: Path | str | None = None,
+        max_concurrent_requests: int = DEFAULT_MAX_CONCURRENT_REQUESTS,
     ):
         """Initialize AWS service fetcher.
 
@@ -130,10 +134,13 @@ class AWSServiceFetcher:
                             instead of making API calls. Directory should contain:
                             - _services.json: List of all services
                             - {service}.json: Individual service files (e.g., s3.json)
+            max_concurrent_requests: Maximum number of concurrent HTTP requests (default: 10)
         """
         self.prefetch_common = prefetch_common
         self.aws_services_dir = Path(aws_services_dir) if aws_services_dir else None
         self._prefetched_services: set[str] = set()
+        # Semaphore for limiting concurrent requests
+        self._request_semaphore = asyncio.Semaphore(max_concurrent_requests)
 
         # Initialize storage component
         self._storage = ServiceFileStorage(
@@ -343,7 +350,10 @@ class AWSServiceFetcher:
         raise ValueError(f"Service `{service_name}` not found")
 
     async def fetch_multiple_services(self, service_names: list[str]) -> dict[str, ServiceDetail]:
-        """Fetch multiple services concurrently with optimized batching.
+        """Fetch multiple services concurrently with controlled parallelism.
+
+        Uses a semaphore to limit concurrent requests and prevent overwhelming
+        the AWS service reference API.
 
         Args:
             service_names: List of service names to fetch
@@ -361,14 +371,16 @@ class AWSServiceFetcher:
         """
 
         async def fetch_single(name: str) -> tuple[str, ServiceDetail]:
-            try:
-                detail = await self.fetch_service_by_name(name)
-                return name, detail
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                logger.error(f"Failed to fetch service {name}: {e}")
-                raise
-
-        # Fetch all services concurrently
+            # Use semaphore to limit concurrent requests
+            async with self._request_semaphore:
+                try:
+                    detail = await self.fetch_service_by_name(name)
+                    return name, detail
+                except Exception as e:  # pylint: disable=broad-exception-caught
+                    logger.error(f"Failed to fetch service {name}: {e}")
+                    raise
+
+        # Fetch all services concurrently (semaphore controls parallelism)
         tasks = [fetch_single(name) for name in service_names]
         results = await asyncio.gather(*tasks, return_exceptions=True)
 

iam_validator/core/codeowners.py

@@ -0,0 +1,245 @@
+"""CODEOWNERS file parser for GitHub repositories.
+
+This module provides functionality to parse GitHub CODEOWNERS files and
+determine which users/teams own specific files. Used to authorize users
+who can ignore validation findings.
+
+Reference: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import PurePosixPath
+from typing import ClassVar
+
+
+@dataclass(slots=True)
+class CodeOwnerRule:
+    """A single rule from a CODEOWNERS file.
+
+    Attributes:
+        pattern: File pattern (glob-style, GitHub CODEOWNERS format)
+        owners: List of @users and/or @org/teams
+        compiled_pattern: Pre-compiled regex for fast matching
+    """
+
+    pattern: str
+    owners: list[str]
+    compiled_pattern: re.Pattern[str] | None = field(default=None, repr=False)
+
+    def __post_init__(self) -> None:
+        """Pre-compile the pattern for efficient matching."""
+        self.compiled_pattern = _compile_codeowners_pattern(self.pattern)
+
+
+@lru_cache(maxsize=256)
+def _compile_codeowners_pattern(pattern: str) -> re.Pattern[str]:
+    """Compile a CODEOWNERS pattern to regex with caching.
+
+    CODEOWNERS patterns follow these rules:
+    - Patterns starting with / are relative to repo root
+    - Patterns without / match any path containing that component
+    - * matches anything except /
+    - ** matches anything including /
+    - Trailing / matches directories
+
+    Args:
+        pattern: CODEOWNERS glob pattern
+
+    Returns:
+        Compiled regex pattern
+    """
+    # Normalize the pattern
+    original_pattern = pattern
+    pattern = pattern.strip()
+
+    # Handle leading slash (anchored to root)
+    anchored = pattern.startswith("/")
+    if anchored:
+        pattern = pattern[1:]
+
+    # Handle trailing slash (directory match)
+    is_dir = pattern.endswith("/")
+    if is_dir:
+        pattern = pattern[:-1]
+
+    # Escape special regex characters except * and ?
+    pattern = re.escape(pattern)
+
+    # Convert glob patterns to regex
+    # ** matches any number of directories
+    pattern = pattern.replace(r"\*\*", "<<<DOUBLE_STAR>>>")
+    # * matches anything except /
+    pattern = pattern.replace(r"\*", "[^/]*")
+    # ** -> match anything
+    pattern = pattern.replace("<<<DOUBLE_STAR>>>", ".*")
+    # ? matches single character except /
+    pattern = pattern.replace(r"\?", "[^/]")
+
+    # Build final regex
+    if anchored:
+        # Anchored patterns match from repo root
+        regex = f"^{pattern}"
+    elif "/" in original_pattern.lstrip("/"):
+        # Patterns with / in them are implicitly anchored
+        regex = f"^{pattern}"
+    else:
+        # Patterns without / can match anywhere in path
+        regex = f"(^|/){pattern}"
+
+    if is_dir:
+        # Directory patterns match the directory and anything under it
+        regex += "(/|$)"
+    else:
+        # File patterns match exactly or as prefix for directories
+        regex += "($|/)"
+
+    return re.compile(regex)
+
+
+class CodeOwnersParser:
+    """Parser for GitHub CODEOWNERS file format.
+
+    Parses CODEOWNERS content and provides file-to-owner mapping.
+    Uses last-matching-pattern semantics as per GitHub's behavior.
+
+    Example:
+        >>> content = '''
+        ... # Default owners
+        ... * @default-team
+        ... # IAM policies owned by security
+        ... /policies/**/*.json @security-team @security-lead
+        ... '''
+        >>> parser = CodeOwnersParser(content)
+        >>> parser.get_owners_for_file("policies/admin/admin.json")
+        ['@security-team', '@security-lead']
+    """
+
+    CODEOWNERS_PATHS: ClassVar[list[str]] = [
+        "CODEOWNERS",
+        ".github/CODEOWNERS",
+        "docs/CODEOWNERS",
+    ]
+
+    def __init__(self, content: str) -> None:
+        """Initialize parser with CODEOWNERS content.
+
+        Args:
+            content: Raw content of CODEOWNERS file
+        """
+        self.rules: list[CodeOwnerRule] = []
+        self._parse(content)
+
+    def _parse(self, content: str) -> None:
+        """Parse CODEOWNERS file content.
+
+        Args:
+            content: Raw CODEOWNERS file content
+        """
+        for line in content.splitlines():
+            line = line.strip()
+
+            # Skip empty lines and comments
+            if not line or line.startswith("#"):
+                continue
+
+            # Split into pattern and owners
+            parts = line.split()
+            if len(parts) >= 2:
+                pattern = parts[0]
+                owners = parts[1:]
+                self.rules.append(CodeOwnerRule(pattern=pattern, owners=owners))
+            elif len(parts) == 1:
+                # Pattern with no owners (unsets ownership)
+                self.rules.append(CodeOwnerRule(pattern=parts[0], owners=[]))
+
+    def get_owners_for_file(self, file_path: str) -> list[str]:
+        """Get owners for a specific file path.
+
+        Uses last-matching-pattern semantics as per GitHub's behavior.
+        If multiple patterns match, the last one in the file wins.
+
+        Args:
+            file_path: Path to the file (relative to repo root)
+
+        Returns:
+            List of owners for the file, or empty list if no match
+        """
+        # Normalize path (remove leading ./ or /)
+        file_path = file_path.lstrip("./")
+
+        # Find all matching rules, last one wins
+        owners: list[str] = []
+        for rule in self.rules:
+            if rule.compiled_pattern and rule.compiled_pattern.search(file_path):
+                owners = rule.owners
+
+        return owners
+
+    def is_owner(self, username: str, file_path: str) -> bool:
+        """Check if a user is an owner of a file.
+
+        Note: This only checks direct username matches. For team membership,
+        use GitHubIntegration.is_user_codeowner() which resolves teams.
+
+        Args:
+            username: GitHub username (with or without @)
+            file_path: Path to the file
+
+        Returns:
+            True if user is directly listed as owner
+        """
+        # Normalize username
+        username = username.lstrip("@").lower()
+
+        owners = self.get_owners_for_file(file_path)
+        for owner in owners:
+            owner = owner.lstrip("@").lower()
+            # Direct username match (not team)
+            if "/" not in owner and owner == username:
+                return True
+
+        return False
+
+    def get_teams_for_file(self, file_path: str) -> list[tuple[str, str]]:
+        """Get team owners for a file as (org, team_slug) tuples.
+
+        Args:
+            file_path: Path to the file
+
+        Returns:
+            List of (org, team_slug) tuples
+        """
+        owners = self.get_owners_for_file(file_path)
+        teams: list[tuple[str, str]] = []
+
+        for owner in owners:
+            owner = owner.lstrip("@")
+            if "/" in owner:
+                parts = owner.split("/", 1)
+                if len(parts) == 2:
+                    teams.append((parts[0], parts[1]))
+
+        return teams
+
+
+def normalize_path(path: str) -> str:
+    """Normalize a file path for CODEOWNERS matching.
+
+    Args:
+        path: File path (may be absolute or relative)
+
+    Returns:
+        Normalized relative path
+    """
+    # Convert to posix-style path
+    path = str(PurePosixPath(path))
+    # Remove leading ./
+    if path.startswith("./"):
+        path = path[2:]
+    # Remove leading /
+    path = path.lstrip("/")
+    return path