cloudg 0.3.0__py3-none-any.whl

cloudg/scanners/trivy.py

@@ -0,0 +1,339 @@
+"""Trivy container and CVE scanner wrapper."""
+
+from __future__ import annotations
+
+import json
+import logging
+import shutil
+import subprocess
+from typing import Any
+
+from cloudg.schema.models import Finding, Severity
+
+logger = logging.getLogger(__name__)
+
+_SEVERITY_MAP = {
+    "CRITICAL": Severity.CRITICAL,
+    "HIGH": Severity.HIGH,
+    "MEDIUM": Severity.MEDIUM,
+    "LOW": Severity.LOW,
+    "UNKNOWN": Severity.INFO,
+}
+
+
+class TrivyScanner:
+    """Wraps Trivy CLI for container image CVE and secret scanning.
+
+    Scans container images from ECR/ACR/GCR and maps CVEs to findings.
+    """
+
+    def __init__(
+        self,
+        extra_args: list[str] | None = None,
+    ) -> None:
+        self._extra_args = extra_args or []
+
+    @staticmethod
+    def is_available() -> bool:
+        """Check if Trivy is installed."""
+        return shutil.which("trivy") is not None
+
+    def scan_image(self, image: str) -> list[Finding]:
+        """Scan a single container image for CVEs.
+
+        Args:
+            image: Full image reference (e.g. 123456.dkr.ecr.us-east-1.amazonaws.com/app:latest).
+
+        Returns:
+            List of Finding objects for discovered CVEs.
+        """
+        if not self.is_available():
+            logger.warning(
+                "Trivy is not installed. Install from: https://trivy.dev/. Skipping Trivy scan."
+            )
+            return []
+
+        cmd = [
+            "trivy",
+            "image",
+            "--format",
+            "json",
+            "--quiet",
+            *self._extra_args,
+            image,
+        ]
+
+        logger.info("Running Trivy: %s", " ".join(cmd))
+
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=1800,
+                check=False,
+            )
+        except subprocess.TimeoutExpired:
+            logger.error("Trivy scan timed out for image %s", image)
+            return []
+        except Exception as exc:
+            logger.error("Failed to run Trivy for %s: %s", image, exc)
+            return []
+
+        return self._parse_output(result.stdout, image)
+
+    def scan_images(self, images: list[str]) -> list[Finding]:
+        """Scan multiple container images.
+
+        Args:
+            images: List of image references.
+
+        Returns:
+            Combined list of Finding objects.
+        """
+        all_findings: list[Finding] = []
+        for image in images:
+            all_findings.extend(self.scan_image(image))
+        return all_findings
+
+    def scan_filesystem(self, directories: list[str]) -> list[Finding]:
+        """Scan filesystem directories for misconfigurations and vulnerabilities.
+
+        Uses 'trivy fs' to scan IaC files, lock files, and other artifacts
+        without requiring container images.
+
+        Args:
+            directories: List of directory paths to scan.
+
+        Returns:
+            Combined list of Finding objects.
+        """
+        if not self.is_available():
+            logger.warning(
+                "Trivy is not installed. Install from: https://trivy.dev/. "
+                "Skipping Trivy filesystem scan."
+            )
+            return []
+
+        all_findings: list[Finding] = []
+        for directory in directories:
+            cmd = [
+                "trivy",
+                "fs",
+                "--format",
+                "json",
+                "--quiet",
+                "--scanners",
+                "vuln,misconfig,secret",
+                *self._extra_args,
+                directory,
+            ]
+
+            logger.info("Running Trivy filesystem scan: %s", " ".join(cmd))
+
+            try:
+                result = subprocess.run(
+                    cmd,
+                    capture_output=True,
+                    text=True,
+                    timeout=1800,
+                    check=False,
+                )
+            except subprocess.TimeoutExpired:
+                logger.error("Trivy filesystem scan timed out for %s", directory)
+                continue
+            except Exception as exc:
+                logger.error("Failed to run Trivy fs for %s: %s", directory, exc)
+                continue
+
+            findings = self._parse_fs_output(result.stdout, directory)
+            all_findings.extend(findings)
+
+        return all_findings
+
+    def _parse_fs_output(self, stdout: str, directory: str) -> list[Finding]:
+        """Parse Trivy filesystem scan JSON output."""
+        findings: list[Finding] = []
+
+        if not stdout.strip():
+            logger.warning("Trivy fs produced no output for %s", directory)
+            return []
+
+        try:
+            data = json.loads(stdout)
+        except json.JSONDecodeError as exc:
+            logger.warning("Failed to parse Trivy fs JSON: %s", exc)
+            return []
+
+        results = data.get("Results", [])
+        for result in results:
+            target = result.get("Target", "")
+            target_type = result.get("Type", "")
+
+            # Vulnerabilities (from lock files, package manifests)
+            for vuln in result.get("Vulnerabilities", []):
+                finding = self._parse_vulnerability(vuln, directory, target, target_type)
+                if finding:
+                    findings.append(finding)
+
+            # Misconfigurations (from IaC files)
+            for misconfig in result.get("Misconfigurations", []):
+                finding = self._parse_misconfig(misconfig, directory, target)
+                if finding:
+                    findings.append(finding)
+
+            # Secret findings
+            for secret in result.get("Secrets", []):
+                finding = self._parse_secret(secret, directory, target)
+                if finding:
+                    findings.append(finding)
+
+        logger.info("Parsed %d findings from Trivy fs for %s", len(findings), directory)
+        return findings
+
+    def _parse_misconfig(
+        self,
+        misconfig: dict[str, Any],
+        directory: str,
+        target: str,
+    ) -> Finding | None:
+        """Parse a single Trivy misconfiguration finding."""
+        try:
+            severity_str = misconfig.get("Severity", "UNKNOWN").upper()
+            severity = _SEVERITY_MAP.get(severity_str, Severity.MEDIUM)
+
+            misconfig_id = misconfig.get("ID", "")
+            title = misconfig.get("Title", "")
+            description = misconfig.get("Description", "")
+            if len(description) > 500:
+                description = description[:500] + "..."
+
+            resolution = misconfig.get("Resolution", "")
+            primary_url = misconfig.get("PrimaryURL", "")
+
+            return Finding(
+                resource_id=directory,
+                resource_arn=directory,
+                severity=severity,
+                title=f"[Trivy/IaC] {misconfig_id}: {title}",
+                description=description,
+                evidence=f"Directory: {directory}, Target: {target}",
+                remediation=resolution
+                if resolution
+                else primary_url or f"See Trivy check {misconfig_id}",
+                source_tool="trivy",
+                source_finding_id=misconfig_id,
+                compliance_frameworks=["CIS"],
+            )
+        except Exception as exc:
+            logger.debug("Failed to parse Trivy misconfiguration: %s", exc)
+            return None
+
+    def _parse_output(self, stdout: str, image: str) -> list[Finding]:
+        """Parse Trivy JSON output."""
+        findings: list[Finding] = []
+
+        if not stdout.strip():
+            return []
+
+        try:
+            data = json.loads(stdout)
+        except json.JSONDecodeError as exc:
+            logger.warning("Failed to parse Trivy JSON: %s", exc)
+            return []
+
+        results = data.get("Results", [])
+        for result in results:
+            target = result.get("Target", "")
+            target_type = result.get("Type", "")
+
+            # CVE vulnerabilities
+            for vuln in result.get("Vulnerabilities", []):
+                finding = self._parse_vulnerability(vuln, image, target, target_type)
+                if finding:
+                    findings.append(finding)
+
+            # Secret findings
+            for secret in result.get("Secrets", []):
+                finding = self._parse_secret(secret, image, target)
+                if finding:
+                    findings.append(finding)
+
+        logger.info("Parsed %d findings from Trivy for %s", len(findings), image)
+        return findings
+
+    def _parse_vulnerability(
+        self,
+        vuln: dict[str, Any],
+        image: str,
+        target: str,
+        target_type: str,
+    ) -> Finding | None:
+        """Parse a single Trivy CVE vulnerability."""
+        try:
+            severity_str = vuln.get("Severity", "UNKNOWN").upper()
+            severity = _SEVERITY_MAP.get(severity_str, Severity.MEDIUM)
+
+            cve_id = vuln.get("VulnerabilityID", "")
+            pkg_name = vuln.get("PkgName", "")
+            installed_version = vuln.get("InstalledVersion", "")
+            fixed_version = vuln.get("FixedVersion", "")
+
+            description = vuln.get("Description", "")
+            if len(description) > 500:
+                description = description[:500] + "..."
+
+            cvss_score = None
+            cvss_data = vuln.get("CVSS", {})
+            for source in cvss_data.values():
+                if isinstance(source, dict) and "V3Score" in source:
+                    cvss_score = source["V3Score"]
+                    break
+
+            remediation = ""
+            if fixed_version:
+                remediation = f"Update {pkg_name} from {installed_version} to {fixed_version}"
+            else:
+                remediation = (
+                    f"No fix available for {cve_id} in {pkg_name}. Consider alternative packages."
+                )
+
+            return Finding(
+                resource_id=image,
+                resource_arn=image,
+                severity=severity,
+                title=f"[Trivy] {cve_id}: {pkg_name} ({target_type})",
+                description=description,
+                evidence=f"Image: {image}, Target: {target}, Package: {pkg_name}@{installed_version}",
+                remediation=remediation,
+                source_tool="trivy",
+                source_finding_id=cve_id,
+                cvss_score=cvss_score,
+                compliance_frameworks=["CVE"],
+            )
+        except Exception as exc:
+            logger.debug("Failed to parse Trivy vulnerability: %s", exc)
+            return None
+
+    def _parse_secret(
+        self,
+        secret: dict[str, Any],
+        image: str,
+        target: str,
+    ) -> Finding | None:
+        """Parse a Trivy secret finding."""
+        try:
+            return Finding(
+                resource_id=image,
+                resource_arn=image,
+                severity=Severity.HIGH,
+                title=f"[Trivy] Secret found in container image: {secret.get('RuleID', 'unknown')}",
+                description=f"Category: {secret.get('Category', '')}, Title: {secret.get('Title', '')}",
+                evidence=f"Image: {image}, Target: {target}, Match: {secret.get('Match', '')[:200]}",
+                remediation="Remove secrets from container images. Use secret management services (AWS Secrets Manager, Azure Key Vault, GCP Secret Manager).",
+                source_tool="trivy",
+                source_finding_id=secret.get("RuleID", ""),
+            )
+        except Exception as exc:
+            logger.debug("Failed to parse Trivy secret: %s", exc)
+            return None

cloudg/schema/__init__.py

@@ -0,0 +1,25 @@
+"""Schema models for cloud asset normalisation."""
+
+from cloudg.schema.models import (
+    AssetType,
+    CloudAsset,
+    CloudProvider,
+    ComplianceResult,
+    ComplianceStatus,
+    Finding,
+    NetworkEdge,
+    ScanResult,
+    Severity,
+)
+
+__all__ = [
+    "AssetType",
+    "CloudAsset",
+    "CloudProvider",
+    "ComplianceResult",
+    "ComplianceStatus",
+    "Finding",
+    "NetworkEdge",
+    "ScanResult",
+    "Severity",
+]

cloudg/schema/models.py

@@ -0,0 +1,260 @@
+"""Pydantic v2 schema models for cloud assets, findings, and relationships."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from enum import Enum
+from typing import Annotated, Any
+
+from pydantic import BaseModel, ConfigDict, Field, computed_field
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+class Severity(str, Enum):
+    """Finding severity levels."""
+
+    CRITICAL = "CRITICAL"
+    HIGH = "HIGH"
+    MEDIUM = "MEDIUM"
+    LOW = "LOW"
+    INFO = "INFO"
+
+
+class CloudProvider(str, Enum):
+    """Supported cloud providers."""
+
+    AWS = "AWS"
+    AZURE = "AZURE"
+    GCP = "GCP"
+
+
+class AssetType(str, Enum):
+    """Normalised asset type taxonomy."""
+
+    # Compute
+    EC2 = "EC2"
+    VIRTUAL_MACHINE = "VIRTUAL_MACHINE"
+    GCE_INSTANCE = "GCE_INSTANCE"
+    LAMBDA_FUNCTION = "LAMBDA_FUNCTION"
+    CLOUD_FUNCTION = "CLOUD_FUNCTION"
+    ECS_CLUSTER = "ECS_CLUSTER"
+    EKS_CLUSTER = "EKS_CLUSTER"
+    AKS_CLUSTER = "AKS_CLUSTER"
+    GKE_CLUSTER = "GKE_CLUSTER"
+    APP_SERVICE = "APP_SERVICE"
+
+    # Networking
+    VPC = "VPC"
+    VNET = "VNET"
+    SUBNET = "SUBNET"
+    SECURITY_GROUP = "SECURITY_GROUP"
+    NSG = "NSG"
+    NACL = "NACL"
+    ROUTE_TABLE = "ROUTE_TABLE"
+    INTERNET_GATEWAY = "INTERNET_GATEWAY"
+    NAT_GATEWAY = "NAT_GATEWAY"
+    LOAD_BALANCER = "LOAD_BALANCER"
+    CLOUDFRONT = "CLOUDFRONT"
+    CDN = "CDN"
+    TRANSIT_GATEWAY = "TRANSIT_GATEWAY"
+    PEERING_CONNECTION = "PEERING_CONNECTION"
+    ELASTIC_IP = "ELASTIC_IP"
+
+    # Storage
+    S3_BUCKET = "S3_BUCKET"
+    BLOB_STORAGE = "BLOB_STORAGE"
+    GCS_BUCKET = "GCS_BUCKET"
+    EBS_VOLUME = "EBS_VOLUME"
+
+    # Database
+    RDS_INSTANCE = "RDS_INSTANCE"
+    AURORA_CLUSTER = "AURORA_CLUSTER"
+    AZURE_SQL = "AZURE_SQL"
+    CLOUD_SQL = "CLOUD_SQL"
+    DYNAMODB_TABLE = "DYNAMODB_TABLE"
+
+    # IAM
+    IAM_USER = "IAM_USER"
+    IAM_ROLE = "IAM_ROLE"
+    IAM_POLICY = "IAM_POLICY"
+    IAM_GROUP = "IAM_GROUP"
+    SERVICE_PRINCIPAL = "SERVICE_PRINCIPAL"
+
+    # Secrets / Keys
+    KMS_KEY = "KMS_KEY"
+    SECRET = "SECRET"
+    CERTIFICATE = "CERTIFICATE"
+    KEY_VAULT = "KEY_VAULT"
+
+    # Logging
+    CLOUDTRAIL = "CLOUDTRAIL"
+    FLOW_LOG = "FLOW_LOG"
+
+    # Generic
+    OTHER = "OTHER"
+
+
+class EdgeType(str, Enum):
+    """Relationship edge types."""
+
+    SECURITY_GROUP_RULE = "SECURITY_GROUP_RULE"
+    NACL_RULE = "NACL_RULE"
+    ROUTE = "ROUTE"
+    IAM_TRUST = "IAM_TRUST"
+    IAM_POLICY_ATTACHMENT = "IAM_POLICY_ATTACHMENT"
+    CONTAINS = "CONTAINS"  # VPC contains Subnet
+    PEERING = "PEERING"
+    LOAD_BALANCER_TARGET = "LOAD_BALANCER_TARGET"
+    INTERNET_EXPOSED = "INTERNET_EXPOSED"
+
+
+class ComplianceStatus(str, Enum):
+    """Compliance check status."""
+
+    PASS = "PASS"
+    FAIL = "FAIL"
+    NOT_APPLICABLE = "NOT_APPLICABLE"
+    MANUAL = "MANUAL"
+
+
+# ---------------------------------------------------------------------------
+# Helper types
+# ---------------------------------------------------------------------------
+
+NonEmptyStr = Annotated[str, Field(min_length=1)]
+
+
+# ---------------------------------------------------------------------------
+# Core Models
+# ---------------------------------------------------------------------------
+
+
+class CloudAsset(BaseModel):
+    """Normalised cloud resource representation."""
+
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    arn: str | None = Field(default=None, description="Cloud-native resource identifier")
+    name: str = Field(description="Human-readable resource name")
+    asset_type: AssetType
+    provider: CloudProvider
+    region: str = Field(default="global")
+    account_id: str | None = None
+    tags: dict[str, str] = Field(default_factory=dict)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    raw_data: dict[str, Any] = Field(default_factory=dict, exclude=True)
+    collected_at: datetime = Field(default_factory=datetime.utcnow)
+    is_internet_exposed: bool = False
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def display_id(self) -> str:
+        """Return the best identifier for display."""
+        return self.arn or self.id
+
+
+class NetworkEdge(BaseModel):
+    """Directed edge representing a network relationship or access path."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    source_id: str = Field(description="Source node ID")
+    target_id: str = Field(description="Target node ID")
+    edge_type: EdgeType
+    ports: list[int] = Field(default_factory=list)
+    port_range: str | None = None  # e.g. "80-443"
+    protocol: str | None = None  # TCP, UDP, ICMP, ALL
+    cidr: str | None = None  # e.g. "0.0.0.0/0"
+    direction: str = "ingress"  # ingress | egress
+    description: str | None = None
+
+
+class Finding(BaseModel):
+    """Normalised security or governance finding."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    resource_id: str = Field(description="Internal asset ID")
+    resource_arn: str | None = None
+    severity: Severity
+    title: str
+    description: str
+    evidence: str | None = None
+    remediation: str | None = None
+    source_tool: str = Field(description="Tool that produced this finding")
+    source_finding_id: str | None = None
+    compliance_frameworks: list[str] = Field(default_factory=list)
+    cvss_score: float | None = Field(default=None, ge=0.0, le=10.0)
+    detected_at: datetime = Field(default_factory=datetime.utcnow)
+    is_suppressed: bool = False
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def risk_score(self) -> float:
+        """Composite risk score (0-10) based on severity + CVSS."""
+        severity_scores = {
+            Severity.CRITICAL: 9.5,
+            Severity.HIGH: 7.5,
+            Severity.MEDIUM: 5.0,
+            Severity.LOW: 2.5,
+            Severity.INFO: 0.5,
+        }
+        base = severity_scores.get(self.severity, 5.0)
+        if self.cvss_score is not None:
+            return round((base + self.cvss_score) / 2, 1)
+        return base
+
+
+class ComplianceResult(BaseModel):
+    """Mapping of a finding to a compliance framework control."""
+
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    framework: str = Field(description="e.g. CIS, NIST-800-53, PCI-DSS, GDPR")
+    control_id: str = Field(description="e.g. CIS-1.1, NIST-AC-2")
+    control_title: str | None = None
+    status: ComplianceStatus
+    finding_ids: list[str] = Field(default_factory=list)
+    resource_arn: str | None = None
+
+
+class ScanResult(BaseModel):
+    """Aggregate pipeline output container."""
+
+    model_config = ConfigDict(extra="allow")
+
+    scan_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
+    provider: CloudProvider | None = None
+    account_id: str | None = None
+    region: str | None = None
+    started_at: datetime = Field(default_factory=datetime.utcnow)
+    completed_at: datetime | None = None
+
+    assets: list[CloudAsset] = Field(default_factory=list)
+    findings: list[Finding] = Field(default_factory=list)
+    edges: list[NetworkEdge] = Field(default_factory=list)
+    compliance: list[ComplianceResult] = Field(default_factory=list)
+
+    @computed_field  # type: ignore[misc]
+    @property
+    def summary(self) -> dict[str, Any]:
+        """Quick summary statistics."""
+        severity_counts: dict[str, int] = {}
+        for f in self.findings:
+            severity_counts[f.severity.value] = severity_counts.get(f.severity.value, 0) + 1
+        return {
+            "total_assets": len(self.assets),
+            "total_findings": len(self.findings),
+            "total_edges": len(self.edges),
+            "severity_breakdown": severity_counts,
+            "compliance_frameworks": list({c.framework for c in self.compliance}),
+        }