cloudg 0.3.0__py3-none-any.whl

cloudg/scanners/iam_linter.py

@@ -0,0 +1,200 @@
+"""IAM policy linter using Parliament and policy-sentry."""
+
+from __future__ import annotations
+
+import json
+import logging
+
+from cloudg.schema.models import CloudAsset, AssetType, Finding, Severity
+
+logger = logging.getLogger(__name__)
+
+
+class IAMLinter:
+    """Analyses IAM policies using Parliament for linting and policy-sentry for least-privilege scoring.
+
+    Parliament: Detects wildcard abuse, logical errors, unused conditions.
+    Policy-sentry: Scores how over-permissioned policies are.
+    """
+
+    def __init__(self) -> None:
+        self._parliament_available = False
+        self._policy_sentry_available = False
+        self._check_availability()
+
+    def _check_availability(self) -> None:
+        """Check which IAM analysis tools are available."""
+        try:
+            import parliament  # noqa: F401
+
+            self._parliament_available = True
+        except ImportError:
+            logger.warning("parliament not installed. Install with: pip install parliament")
+
+        try:
+            import policy_sentry  # noqa: F401
+
+            self._policy_sentry_available = True
+        except ImportError:
+            logger.debug("policy-sentry not installed. Install with: pip install policy-sentry")
+
+    def analyze_policies(self, assets: list[CloudAsset]) -> list[Finding]:
+        """Analyse all IAM policy assets.
+
+        Args:
+            assets: List of cloud assets (filters for IAM_POLICY and IAM_ROLE types).
+
+        Returns:
+            List of Finding objects for IAM policy issues.
+        """
+        findings: list[Finding] = []
+
+        iam_assets = [
+            a for a in assets if a.asset_type in (AssetType.IAM_POLICY, AssetType.IAM_ROLE)
+        ]
+
+        for asset in iam_assets:
+            policy_doc = asset.metadata.get("assume_role_policy") or asset.metadata.get(
+                "policy_document"
+            )
+            if not policy_doc:
+                continue
+
+            # Ensure it's a JSON string
+            if isinstance(policy_doc, dict):
+                policy_str = json.dumps(policy_doc)
+            else:
+                policy_str = str(policy_doc)
+
+            # Run Parliament linting
+            if self._parliament_available:
+                findings.extend(self._lint_with_parliament(policy_str, asset))
+
+            # Check for wildcard permissions
+            findings.extend(self._check_wildcards(policy_str, asset))
+
+        logger.info("IAM linter produced %d findings", len(findings))
+        return findings
+
+    def _lint_with_parliament(self, policy_str: str, asset: CloudAsset) -> list[Finding]:
+        """Run Parliament linter on a policy document."""
+        findings: list[Finding] = []
+
+        try:
+            import parliament
+
+            analyzed = parliament.analyze_policy_string(policy_str)
+
+            for finding in analyzed.findings:
+                severity = self._map_parliament_severity(finding.severity)
+                findings.append(
+                    Finding(
+                        resource_id=asset.id,
+                        resource_arn=asset.arn,
+                        severity=severity,
+                        title=f"[Parliament] IAM policy issue: {finding.issue}",
+                        description=finding.detail if hasattr(finding, "detail") else str(finding),
+                        evidence=f"Policy: {asset.name}, Issue: {finding.issue}",
+                        remediation=(
+                            "Review and fix the IAM policy. Follow least-privilege "
+                            "principles. Use policy-sentry to generate scoped policies."
+                        ),
+                        source_tool="parliament",
+                        compliance_frameworks=["CIS", "NIST-800-53"],
+                    )
+                )
+        except Exception as exc:
+            logger.debug("Parliament analysis failed for %s: %s", asset.name, exc)
+
+        return findings
+
+    def _check_wildcards(self, policy_str: str, asset: CloudAsset) -> list[Finding]:
+        """Check for wildcard permissions (Action: * or Resource: *)."""
+        findings: list[Finding] = []
+
+        try:
+            policy = json.loads(policy_str)
+        except json.JSONDecodeError:
+            return findings
+
+        statements = policy.get("Statement", [])
+        if isinstance(statements, dict):
+            statements = [statements]
+
+        for statement in statements:
+            effect = statement.get("Effect", "")
+            actions = statement.get("Action", [])
+            resources = statement.get("Resource", [])
+
+            if isinstance(actions, str):
+                actions = [actions]
+            if isinstance(resources, str):
+                resources = [resources]
+
+            # Check for wildcard actions
+            if effect == "Allow" and "*" in actions:
+                findings.append(
+                    Finding(
+                        resource_id=asset.id,
+                        resource_arn=asset.arn,
+                        severity=Severity.HIGH,
+                        title=f"Wildcard Action (*) in IAM policy: {asset.name}",
+                        description=(
+                            f"The IAM policy '{asset.name}' grants 'Action: *' "
+                            f"(all actions). This violates least-privilege principles."
+                        ),
+                        evidence=f"Statement: {json.dumps(statement)[:500]}",
+                        remediation=(
+                            "Replace 'Action: *' with specific actions required "
+                            "by the workload. Use AWS Access Advisor to identify "
+                            "unused permissions."
+                        ),
+                        source_tool="cloudg-iam",
+                        compliance_frameworks=["CIS", "NIST-800-53", "SOC2"],
+                    )
+                )
+
+            # Check for wildcard resources with sensitive actions
+            if effect == "Allow" and "*" in resources:
+                sensitive_prefixes = ("iam:", "sts:", "kms:", "s3:", "ec2:", "lambda:")
+                has_sensitive = any(
+                    any(a.startswith(p) for p in sensitive_prefixes)
+                    for a in actions
+                    if isinstance(a, str) and a != "*"
+                )
+
+                if has_sensitive or "*" in actions:
+                    findings.append(
+                        Finding(
+                            resource_id=asset.id,
+                            resource_arn=asset.arn,
+                            severity=Severity.HIGH,
+                            title=f"Wildcard Resource (*) in IAM policy: {asset.name}",
+                            description=(
+                                f"The IAM policy '{asset.name}' grants access to "
+                                f"'Resource: *' (all resources) for sensitive actions."
+                            ),
+                            evidence=f"Statement: {json.dumps(statement)[:500]}",
+                            remediation=(
+                                "Scope 'Resource' to specific ARN patterns. "
+                                "Use conditions to further restrict access."
+                            ),
+                            source_tool="cloudg-iam",
+                            compliance_frameworks=["CIS", "NIST-800-53"],
+                        )
+                    )
+
+        return findings
+
+    @staticmethod
+    def _map_parliament_severity(severity: str) -> Severity:
+        """Map Parliament severity to our Severity enum."""
+        mapping = {
+            "CRITICAL": Severity.CRITICAL,
+            "HIGH": Severity.HIGH,
+            "MEDIUM": Severity.MEDIUM,
+            "LOW": Severity.LOW,
+            "INFO": Severity.INFO,
+            "WARNING": Severity.MEDIUM,
+        }
+        return mapping.get(str(severity).upper(), Severity.MEDIUM)

cloudg/scanners/prowler.py

@@ -0,0 +1,237 @@
+"""Prowler CSPM scanner wrapper — runs Prowler CLI and parses ASFF JSON output."""
+
+from __future__ import annotations
+
+import glob
+import json
+import logging
+import shutil
+import subprocess
+import tempfile
+from typing import Any
+
+from cloudg.schema.models import Finding, Severity
+
+logger = logging.getLogger(__name__)
+
+# Map Prowler severity labels to our enum
+_SEVERITY_MAP = {
+    "critical": Severity.CRITICAL,
+    "high": Severity.HIGH,
+    "medium": Severity.MEDIUM,
+    "low": Severity.LOW,
+    "informational": Severity.INFO,
+    "info": Severity.INFO,
+}
+
+# Compliance framework mapping from Prowler check prefixes
+_COMPLIANCE_MAP = {
+    "cis": "CIS",
+    "nist": "NIST-800-53",
+    "pci": "PCI-DSS",
+    "gdpr": "GDPR",
+    "hipaa": "HIPAA",
+    "soc2": "SOC2",
+}
+
+
+class ProwlerScanner:
+    """Wraps Prowler CLI to run cloud security checks and parse ASFF JSON output.
+
+    Usage:
+        scanner = ProwlerScanner(provider="aws", profile="default")
+        findings = scanner.run()
+    """
+
+    def __init__(
+        self,
+        provider: str = "aws",
+        profile: str | None = None,
+        output_dir: str | None = None,
+        extra_args: list[str] | None = None,
+        aws_access_key_id: str | None = None,
+        aws_secret_access_key: str | None = None,
+        aws_region: str | None = None,
+    ) -> None:
+        self._provider = provider
+        self._profile = profile
+        self._output_dir = output_dir or tempfile.mkdtemp(prefix="prowler_")
+        self._extra_args = extra_args or []
+        self._aws_access_key_id = aws_access_key_id
+        self._aws_secret_access_key = aws_secret_access_key
+        self._aws_region = aws_region
+
+    @staticmethod
+    def is_available() -> bool:
+        """Check if Prowler CLI is installed."""
+        return shutil.which("prowler") is not None
+
+    def run(self) -> list[Finding]:
+        """Execute Prowler scan and return normalised findings.
+
+        Streams stdout/stderr live for real-time progress output.
+
+        Returns:
+            List of Finding objects parsed from Prowler ASFF JSON output.
+        """
+        import os
+        import time
+        import threading
+
+        if not self.is_available():
+            logger.warning(
+                "Prowler is not installed. Install with: pip install prowler. "
+                "Skipping Prowler scan."
+            )
+            return []
+
+        cmd = [
+            "prowler",
+            self._provider,
+            "-M",
+            "json-asff",
+            "-o",
+            self._output_dir,
+        ]
+
+        if self._profile and self._provider == "aws":
+            cmd.extend(["-p", self._profile])
+
+        cmd.extend(self._extra_args)
+
+        logger.info("[Prowler] Starting: %s", " ".join(cmd))
+
+        # Build environment with AWS credentials for the subprocess
+        env = os.environ.copy()
+        if self._aws_access_key_id:
+            env["AWS_ACCESS_KEY_ID"] = self._aws_access_key_id
+        if self._aws_secret_access_key:
+            env["AWS_SECRET_ACCESS_KEY"] = self._aws_secret_access_key
+        if self._aws_region:
+            env["AWS_DEFAULT_REGION"] = self._aws_region
+
+        start_time = time.time()
+
+        try:
+            # Use Popen for live streaming instead of blocking subprocess.run
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                env=env,
+                bufsize=1,  # line-buffered
+            )
+
+            # Stream output in a reader thread
+            def _stream_output():
+                check_count = 0
+                for line in proc.stdout:
+                    stripped = line.rstrip()
+                    if not stripped:
+                        continue
+                    # Count checks for progress reporting
+                    if "PASS" in stripped or "FAIL" in stripped or "WARNING" in stripped:
+                        check_count += 1
+                        if check_count % 25 == 0:
+                            elapsed = int(time.time() - start_time)
+                            logger.info(
+                                "[Prowler] %d checks completed (%ds elapsed)", check_count, elapsed
+                            )
+                    elif "Executing" in stripped or "Service" in stripped:
+                        logger.info("[Prowler] %s", stripped[:120])
+
+            reader = threading.Thread(target=_stream_output, daemon=True)
+            reader.start()
+
+            proc.wait(timeout=3600)
+            reader.join(timeout=5)
+
+            elapsed = int(time.time() - start_time)
+            if proc.returncode != 0:
+                logger.warning("[Prowler] Exited with code %d after %ds", proc.returncode, elapsed)
+            else:
+                logger.info("[Prowler] Completed successfully in %ds", elapsed)
+
+        except subprocess.TimeoutExpired:
+            logger.error("[Prowler] Scan timed out after 3600s — killing process")
+            proc.kill()
+            return []
+        except Exception as exc:
+            logger.error("[Prowler] Failed to run: %s", exc)
+            return []
+
+        return self._parse_output()
+
+    def _parse_output(self) -> list[Finding]:
+        """Parse Prowler ASFF JSON output files."""
+        findings: list[Finding] = []
+        output_files = glob.glob(f"{self._output_dir}/**/*.json", recursive=True)
+
+        for file_path in output_files:
+            try:
+                with open(file_path) as f:
+                    content = f.read().strip()
+                    if not content:
+                        continue
+
+                    # Prowler may output JSON array or JSONL
+                    if content.startswith("["):
+                        data = json.loads(content)
+                    else:
+                        data = [json.loads(line) for line in content.splitlines() if line.strip()]
+
+                    for item in data:
+                        finding = self._parse_asff_finding(item)
+                        if finding:
+                            findings.append(finding)
+
+            except (json.JSONDecodeError, OSError) as exc:
+                logger.warning("Failed to parse Prowler output %s: %s", file_path, exc)
+
+        logger.info("Parsed %d findings from Prowler", len(findings))
+        return findings
+
+    def _parse_asff_finding(self, asff: dict[str, Any]) -> Finding | None:
+        """Convert a single ASFF finding to our Finding model."""
+        try:
+            # Extract severity
+            severity_label = (
+                asff.get("Severity", {}).get("Label", "")
+                or asff.get("ProductFields", {}).get("Severity", "")
+            ).lower()
+            severity = _SEVERITY_MAP.get(severity_label, Severity.MEDIUM)
+
+            # Extract resource ARN
+            resources = asff.get("Resources", [])
+            resource_arn = resources[0].get("Id", "") if resources else ""
+
+            # Extract compliance frameworks
+            compliance = []
+            compliance_data = asff.get("Compliance", {})
+            for standard in compliance_data.get("RelatedRequirements", []):
+                for prefix, framework in _COMPLIANCE_MAP.items():
+                    if prefix in standard.lower():
+                        compliance.append(framework)
+                        break
+
+            # Extract status — skip PASS findings
+            status = asff.get("Compliance", {}).get("Status", "")
+            if status.upper() == "PASSED":
+                return None
+
+            return Finding(
+                resource_id=resource_arn,
+                resource_arn=resource_arn,
+                severity=severity,
+                title=asff.get("Title", "Unknown Prowler Finding"),
+                description=asff.get("Description", ""),
+                evidence=json.dumps(asff.get("ProductFields", {}), default=str)[:1000],
+                remediation=(asff.get("Remediation", {}).get("Recommendation", {}).get("Text", "")),
+                source_tool="prowler",
+                source_finding_id=asff.get("Id", ""),
+                compliance_frameworks=list(set(compliance)),
+            )
+        except Exception as exc:
+            logger.debug("Failed to parse ASFF finding: %s", exc)
+            return None

cloudg/scanners/scoutsuite.py

@@ -0,0 +1,163 @@
+"""ScoutSuite multi-cloud audit scanner wrapper."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from cloudg.schema.models import Finding, Severity
+
+logger = logging.getLogger(__name__)
+
+_SEVERITY_MAP = {
+    "danger": Severity.CRITICAL,
+    "warning": Severity.HIGH,
+    "caution": Severity.MEDIUM,
+    "good": Severity.INFO,
+}
+
+
+class ScoutSuiteScanner:
+    """Wraps ScoutSuite CLI — runs `scout` and parses the results JS file.
+
+    ScoutSuite produces a `scoutsuite_results.js` file containing a JSON
+    object assigned to a JS variable. We strip the variable assignment
+    and parse the JSON.
+    """
+
+    def __init__(
+        self,
+        provider: str = "aws",
+        profile: str | None = None,
+        report_dir: str | None = None,
+        extra_args: list[str] | None = None,
+    ) -> None:
+        self._provider = provider
+        self._profile = profile
+        self._report_dir = report_dir or tempfile.mkdtemp(prefix="scoutsuite_")
+        self._extra_args = extra_args or []
+
+    @staticmethod
+    def is_available() -> bool:
+        """Check if ScoutSuite CLI is installed."""
+        return shutil.which("scout") is not None
+
+    def run(self) -> list[Finding]:
+        """Execute ScoutSuite scan and return normalised findings."""
+        if not self.is_available():
+            logger.warning(
+                "ScoutSuite is not installed. Install with: pip install scoutsuite. "
+                "Skipping ScoutSuite scan."
+            )
+            return []
+
+        cmd = ["scout", self._provider, "--report-dir", self._report_dir, "--no-browser"]
+
+        if self._profile and self._provider == "aws":
+            cmd.extend(["--profile", self._profile])
+
+        cmd.extend(self._extra_args)
+
+        logger.info("Running ScoutSuite: %s", " ".join(cmd))
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=3600,
+                check=False,
+            )
+            if result.returncode != 0:
+                logger.warning(
+                    "ScoutSuite exited with code %d: %s",
+                    result.returncode,
+                    result.stderr[:500],
+                )
+        except subprocess.TimeoutExpired:
+            logger.error("ScoutSuite scan timed out")
+            return []
+        except Exception as exc:
+            logger.error("Failed to run ScoutSuite: %s", exc)
+            return []
+
+        return self._parse_output()
+
+    def _parse_output(self) -> list[Finding]:
+        """Parse ScoutSuite results JS file."""
+        findings: list[Finding] = []
+
+        report_dir = Path(self._report_dir)
+        # ScoutSuite outputs: scoutsuite-results/scoutsuite_results*.js
+        results_files = list(report_dir.rglob("scoutsuite_results*.js"))
+
+        if not results_files:
+            logger.warning("No ScoutSuite results found in %s", self._report_dir)
+            return []
+
+        for results_file in results_files:
+            try:
+                content = results_file.read_text()
+
+                # Strip JS variable assignment: scoutsuite_results = {...}
+                # The JSON object starts after the first '='
+                match = re.search(r"=\s*({.*})\s*;?\s*$", content, re.DOTALL)
+                if not match:
+                    logger.warning("Could not extract JSON from %s", results_file)
+                    continue
+
+                data = json.loads(match.group(1))
+                findings.extend(self._extract_findings(data))
+
+            except (json.JSONDecodeError, OSError) as exc:
+                logger.warning("Failed to parse ScoutSuite results: %s", exc)
+
+        logger.info("Parsed %d findings from ScoutSuite", len(findings))
+        return findings
+
+    def _extract_findings(self, data: dict[str, Any]) -> list[Finding]:
+        """Extract findings from parsed ScoutSuite JSON data."""
+        findings: list[Finding] = []
+
+        # ScoutSuite organises findings by service
+        services = data.get("services", {})
+
+        for service_name, service_data in services.items():
+            if not isinstance(service_data, dict):
+                continue
+
+            findings_data = service_data.get("findings", {})
+            for finding_key, finding_data in findings_data.items():
+                if not isinstance(finding_data, dict):
+                    continue
+
+                flagged_items = finding_data.get("flagged_items", 0)
+                if flagged_items == 0:
+                    continue
+
+                severity_str = finding_data.get("level", "warning").lower()
+                severity = _SEVERITY_MAP.get(severity_str, Severity.MEDIUM)
+
+                items = finding_data.get("items", [])
+                for item in items:
+                    findings.append(
+                        Finding(
+                            resource_id=item if isinstance(item, str) else str(item),
+                            resource_arn=item if isinstance(item, str) else "",
+                            severity=severity,
+                            title=f"[ScoutSuite] {service_name}: {finding_data.get('description', finding_key)}",
+                            description=finding_data.get("rationale", ""),
+                            remediation=finding_data.get("remediation", ""),
+                            source_tool="scoutsuite",
+                            source_finding_id=finding_key,
+                            compliance_frameworks=finding_data.get("references", []),
+                        )
+                    )
+
+        return findings