cloudg 0.3.0__py3-none-any.whl

cloudg/collectors/multi.py

@@ -0,0 +1,329 @@
+"""Multi-provider, multi-account, multi-region collection orchestrator.
+
+Supports simultaneous scanning of AWS + Azure + GCP with:
+- Region auto-discovery via RegionDiscovery
+- Parallel provider execution via asyncio.gather
+- Per-region iteration for Azure and GCP
+- Coverage tracking per provider/region/account
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from typing import Any
+
+from cloudg.config import CloudGConfig
+from cloudg.coverage import CollectionCoverage, ServiceStatus
+from cloudg.region_discovery import RegionDiscovery, is_all_regions
+from cloudg.schema.models import CloudAsset, NetworkEdge
+
+logger = logging.getLogger(__name__)
+
+
+class MultiAccountCollector:
+    """Orchestrates collection across multiple providers, accounts, and regions.
+
+    For AWS: assumes IAM roles in target accounts via STS, iterates regions.
+    For Azure: iterates subscriptions × locations.
+    For GCP: iterates projects × regions.
+
+    Uses asyncio.Semaphore to limit concurrent API calls.
+    """
+
+    def __init__(self, config: CloudGConfig) -> None:
+        self._config = config
+        self._semaphore = asyncio.Semaphore(config.concurrency_limit)
+        self._coverage: list[CollectionCoverage] = []
+        self._region_discovery = RegionDiscovery()
+        self._resolved_regions: dict[str, list[str]] = {}
+
+    async def collect_all(
+        self,
+    ) -> tuple[list[CloudAsset], list[NetworkEdge], list[CollectionCoverage]]:
+        """Collect assets and edges from ALL configured providers simultaneously.
+
+        Returns:
+            Tuple of (all_assets, all_edges, coverage_records).
+        """
+        providers = self._config.providers
+        all_assets: list[CloudAsset] = []
+        all_edges: list[NetworkEdge] = []
+
+        # Build tasks for each provider
+        tasks: list[asyncio.Task] = []
+        task_labels: list[str] = []
+
+        if "aws" in providers:
+            tasks.append(asyncio.ensure_future(self._collect_aws_multi()))
+            task_labels.append("aws")
+
+        if "azure" in providers:
+            tasks.append(asyncio.ensure_future(self._collect_azure_multi()))
+            task_labels.append("azure")
+
+        if "gcp" in providers:
+            tasks.append(asyncio.ensure_future(self._collect_gcp_multi()))
+            task_labels.append("gcp")
+
+        if not tasks:
+            logger.warning("No providers configured for collection")
+            return all_assets, all_edges, self._coverage
+
+        # Execute all providers in parallel
+        logger.info("Starting multi-provider collection for: %s", ", ".join(task_labels))
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for label, result in zip(task_labels, results):
+            if isinstance(result, Exception):
+                logger.error("Provider %s collection failed: %s", label, result)
+                coverage = CollectionCoverage(provider=label)
+                coverage.record(f"{label}_full", ServiceStatus.FAILED, error=str(result))
+                self._coverage.append(coverage)
+            elif isinstance(result, list):
+                for assets, edges in result:
+                    all_assets.extend(assets)
+                    all_edges.extend(edges)
+
+        logger.info(
+            "Multi-provider collection complete: %d assets, %d edges across %d coverage records (%s)",
+            len(all_assets),
+            len(all_edges),
+            len(self._coverage),
+            ", ".join(task_labels),
+        )
+        return all_assets, all_edges, self._coverage
+
+    # ------------------------------------------------------------------
+    # AWS
+    # ------------------------------------------------------------------
+
+    async def _collect_aws_multi(self) -> list[tuple[list[CloudAsset], list[NetworkEdge]]]:
+        """Iterate AWS accounts × regions."""
+        cfg = self._config.aws
+        accounts = cfg.accounts or [None]  # None = current account
+
+        # Region resolution
+        if is_all_regions(cfg.regions):
+            logger.info("AWS: discovering all regions...")
+            regions = await self._region_discovery.discover_aws()
+        else:
+            regions = cfg.regions
+
+        self._resolved_regions["aws"] = regions
+        logger.info("AWS: scanning %d regions × %d accounts", len(regions), len(accounts))
+
+        tasks = []
+        for account_id in accounts:
+            for region in regions:
+                tasks.append(self._collect_aws_single(account_id, region, cfg))
+
+        return await asyncio.gather(*tasks, return_exceptions=False)
+
+    async def _collect_aws_single(
+        self,
+        account_id: str | None,
+        region: str,
+        cfg: Any,
+    ) -> tuple[list[CloudAsset], list[NetworkEdge]]:
+        """Collect from a single AWS account/region.
+
+        Credential priority:
+        1. Direct access keys (access_key_id + secret_access_key) from config
+        2. AWS profile from config
+        3. Environment defaults (AWS_PROFILE, instance metadata, etc.)
+
+        Role assumption is only used when role_name is set AND account_id is specified.
+        """
+        coverage = CollectionCoverage(provider="aws", region=region, account_id=account_id)
+        self._coverage.append(coverage)
+
+        async with self._semaphore:
+            try:
+                from cloudg.credentials import build_aws_session
+
+                # Supports direct keys, OIDC web identity, profiles, the
+                # default chain (instance/task roles), and AssumeRole with
+                # optional ExternalId — see cloudg.credentials.
+                try:
+                    session = build_aws_session(cfg, region, account_id=account_id)
+                except RuntimeError as exc:
+                    logger.error("AWS auth failed for %s/%s: %s", account_id, region, exc)
+                    coverage.record("sts_assume_role", ServiceStatus.FAILED, error=str(exc))
+                    return [], []
+
+                # Resolve account ID if not provided
+                if not account_id:
+                    try:
+                        account_id = session.client("sts").get_caller_identity().get("Account")
+                    except Exception:
+                        account_id = "unknown"
+
+                from cloudg.collectors.aws import AsyncAWSCollector
+
+                collector = AsyncAWSCollector(session=session, region=region, account_id=account_id)
+
+                start = time.time()
+                assets = await collector.collect()
+                edges = await collector.collect_edges()
+                duration_ms = int((time.time() - start) * 1000)
+
+                coverage.record(
+                    "aws_full",
+                    ServiceStatus.SUCCESS,
+                    asset_count=len(assets),
+                    duration_ms=duration_ms,
+                )
+                return assets, edges
+
+            except Exception as exc:
+                logger.error("AWS collection failed for %s/%s: %s", account_id, region, exc)
+                coverage.record("aws_full", ServiceStatus.FAILED, error=str(exc))
+                return [], []
+
+    # ------------------------------------------------------------------
+    # Azure
+    # ------------------------------------------------------------------
+
+    async def _collect_azure_multi(self) -> list[tuple[list[CloudAsset], list[NetworkEdge]]]:
+        """Iterate Azure subscriptions × locations."""
+        sub_ids = self._config.azure.subscription_ids
+        if not sub_ids:
+            sub_ids = [None]
+
+        # Region resolution
+        azure_regions = self._config.azure.regions
+        if is_all_regions(azure_regions):
+            logger.info("Azure: discovering all locations...")
+            first_sub = sub_ids[0] if sub_ids[0] else None
+            try:
+                from cloudg.credentials import build_azure_credential
+
+                cred = build_azure_credential(self._config.azure)
+            except ImportError:
+                cred = None
+            azure_regions = await self._region_discovery.discover_azure(cred, first_sub)
+
+        self._resolved_regions["azure"] = azure_regions
+        logger.info(
+            "Azure: scanning %d subscriptions (locations auto-handled by SDK)", len(sub_ids)
+        )
+
+        # Azure SDKs already return resources across all locations within a subscription.
+        # We pass the resolved regions as metadata but don't iterate per-region for Azure
+        # since list_all() calls return resources from ALL locations.
+        tasks = [self._collect_azure_single(sid) for sid in sub_ids]
+        return await asyncio.gather(*tasks, return_exceptions=False)
+
+    async def _collect_azure_single(
+        self, subscription_id: str | None
+    ) -> tuple[list[CloudAsset], list[NetworkEdge]]:
+        """Collect from a single Azure subscription."""
+        coverage = CollectionCoverage(provider="azure", account_id=subscription_id)
+        self._coverage.append(coverage)
+
+        async with self._semaphore:
+            try:
+                from cloudg.collectors.azure import AzureCollector
+                from cloudg.credentials import build_azure_credential
+
+                # Supports workload identity, service principal (secret or
+                # certificate), managed identity, and the default chain.
+                credential = build_azure_credential(self._config.azure)
+                if not subscription_id:
+                    from azure.mgmt.resource import SubscriptionClient
+
+                    sub_client = SubscriptionClient(credential)
+                    sub = next(sub_client.subscriptions.list(), None)
+                    subscription_id = sub.subscription_id if sub else ""
+
+                collector = AzureCollector(credential=credential, subscription_id=subscription_id)
+
+                start = time.time()
+                assets = await collector.collect()
+                edges = await collector.collect_edges()
+                duration_ms = int((time.time() - start) * 1000)
+
+                coverage.record(
+                    "azure_full",
+                    ServiceStatus.SUCCESS,
+                    asset_count=len(assets),
+                    duration_ms=duration_ms,
+                )
+                return assets, edges
+
+            except Exception as exc:
+                logger.error("Azure collection failed for %s: %s", subscription_id, exc)
+                coverage.record("azure_full", ServiceStatus.FAILED, error=str(exc))
+                return [], []
+
+    # ------------------------------------------------------------------
+    # GCP
+    # ------------------------------------------------------------------
+
+    async def _collect_gcp_multi(self) -> list[tuple[list[CloudAsset], list[NetworkEdge]]]:
+        """Iterate GCP projects × regions."""
+        project_ids = self._config.gcp.project_ids
+        if not project_ids:
+            project_ids = [None]
+
+        # Region resolution
+        gcp_regions = self._config.gcp.regions
+        if is_all_regions(gcp_regions):
+            logger.info("GCP: discovering all regions...")
+            try:
+                from cloudg.credentials import build_gcp_credentials
+
+                creds, default_project = build_gcp_credentials(self._config.gcp)
+                pid = project_ids[0] or default_project
+            except Exception:
+                creds, pid = None, None
+            gcp_regions = await self._region_discovery.discover_gcp(creds, pid)
+
+        self._resolved_regions["gcp"] = gcp_regions
+        logger.info(
+            "GCP: scanning %d projects (Cloud Asset Inventory scans all regions)", len(project_ids)
+        )
+
+        # GCP Cloud Asset Inventory API returns resources across ALL regions for a project.
+        # No per-region iteration needed — the API is project-scoped.
+        tasks = [self._collect_gcp_single(pid) for pid in project_ids]
+        return await asyncio.gather(*tasks, return_exceptions=False)
+
+    async def _collect_gcp_single(
+        self, project_id: str | None
+    ) -> tuple[list[CloudAsset], list[NetworkEdge]]:
+        """Collect from a single GCP project."""
+        coverage = CollectionCoverage(provider="gcp", account_id=project_id)
+        self._coverage.append(coverage)
+
+        async with self._semaphore:
+            try:
+                from cloudg.collectors.gcp import GCPCollector
+                from cloudg.credentials import build_gcp_credentials
+
+                # Supports key files, workload identity federation, ADC
+                # (inherited GCE/GKE identity), and impersonation.
+                credentials, default_project = build_gcp_credentials(self._config.gcp)
+                pid = project_id or default_project
+
+                collector = GCPCollector(project_id=pid, credentials=credentials)
+
+                start = time.time()
+                assets = await collector.collect()
+                edges = await collector.collect_edges()
+                duration_ms = int((time.time() - start) * 1000)
+
+                coverage.record(
+                    "gcp_full",
+                    ServiceStatus.SUCCESS,
+                    asset_count=len(assets),
+                    duration_ms=duration_ms,
+                )
+                return assets, edges
+
+            except Exception as exc:
+                logger.error("GCP collection failed for %s: %s", project_id, exc)
+                coverage.record("gcp_full", ServiceStatus.FAILED, error=str(exc))
+                return [], []

cloudg/config.py

@@ -0,0 +1,261 @@
+"""YAML-based configuration with Pydantic validation."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class AWSConfig(BaseModel):
+    """AWS-specific configuration.
+
+    Auth priority: direct keys → OIDC web identity (role_arn +
+    web_identity_token_file) → profile → default chain (env vars, EC2/ECS
+    instance role, SSO cache). role_arn or accounts[]+role_name adds an
+    STS AssumeRole hop on top, with optional external_id.
+    """
+
+    regions: list[str] = Field(default=["us-east-1"])
+    accounts: list[str] = Field(
+        default_factory=list, description="Account IDs for multi-account scanning"
+    )
+    access_key_id: str | None = Field(
+        default=None, description="AWS access key ID (direct credential)"
+    )
+    secret_access_key: str | None = Field(
+        default=None, description="AWS secret access key (direct credential)"
+    )
+    session_token: str | None = Field(
+        default=None, description="AWS session token (for temporary credentials)"
+    )
+    profile: str | None = Field(
+        default=None, description="AWS CLI profile name (fallback if no direct keys)"
+    )
+    role_name: str | None = Field(
+        default=None,
+        description="IAM role to assume in target accounts (optional, for cross-account)",
+    )
+    role_arn: str | None = Field(
+        default=None,
+        description="Full ARN of a role to assume (single-account AssumeRole or OIDC target)",
+    )
+    external_id: str | None = Field(
+        default=None, description="ExternalId for AssumeRole (third-party auditor pattern)"
+    )
+    web_identity_token_file: str | None = Field(
+        default=None,
+        description="Path to an OIDC token file for AssumeRoleWithWebIdentity (GitHub Actions, EKS, GitLab CI)",
+    )
+    role_session_name: str = Field(
+        default="cloudg-scan", description="STS session name for assumed roles"
+    )
+    max_retries: int = Field(default=10, ge=1, le=30)
+    retry_mode: str = Field(default="adaptive", pattern="^(legacy|standard|adaptive)$")
+
+
+class AzureConfig(BaseModel):
+    """Azure-specific configuration.
+
+    Auth priority: workload identity federation (tenant_id + client_id +
+    federated_token_file) → service principal secret → service principal
+    certificate → managed identity → DefaultAzureCredential chain.
+    """
+
+    subscription_ids: list[str] = Field(default_factory=list)
+    tenant_id: str | None = None
+    client_id: str | None = Field(
+        default=None, description="Service principal / workload identity application (client) ID"
+    )
+    client_secret: str | None = Field(default=None, description="Service principal client secret")
+    certificate_path: str | None = Field(
+        default=None, description="Path to a service principal certificate (PEM/PKCS12)"
+    )
+    federated_token_file: str | None = Field(
+        default=None, description="Path to a federated OIDC token file (workload identity)"
+    )
+    use_managed_identity: bool = Field(
+        default=False, description="Authenticate with the Azure managed identity of the host"
+    )
+    managed_identity_client_id: str | None = Field(
+        default=None, description="Client ID of a user-assigned managed identity"
+    )
+    regions: list[str] = Field(
+        default=["ALL"],
+        description="Azure locations to scan. Use ['ALL'] for auto-discovery.",
+    )
+
+
+class GCPConfig(BaseModel):
+    """GCP-specific configuration.
+
+    Auth priority: credentials_file (service account key or workload
+    identity federation config) → application default credentials
+    (GOOGLE_APPLICATION_CREDENTIALS, gcloud, GCE/GKE metadata).
+    impersonate_service_account adds an impersonation hop on top.
+    """
+
+    project_ids: list[str] = Field(default_factory=list)
+    organization_id: str | None = None
+    credentials_file: str | None = Field(
+        default=None,
+        description="Path to a service account key JSON or external_account (workload identity federation) JSON",
+    )
+    impersonate_service_account: str | None = Field(
+        default=None, description="Service account email to impersonate"
+    )
+    regions: list[str] = Field(
+        default=["ALL"],
+        description="GCP regions to scan. Use ['ALL'] for auto-discovery.",
+    )
+
+
+class ScannerConfig(BaseModel):
+    """Scanner selection and configuration."""
+
+    enabled: list[str] = Field(default=["prowler", "scoutsuite", "checkov", "trivy", "iam"])
+    prowler_extra_args: list[str] = Field(default_factory=list)
+    scoutsuite_extra_args: list[str] = Field(default_factory=list)
+    checkov_extra_args: list[str] = Field(default_factory=list)
+    trivy_extra_args: list[str] = Field(default_factory=list)
+    checkov_frameworks: list[str] = Field(
+        default_factory=list,
+        description="Checkov frameworks to scan. Empty = auto-detect all (recommended for cloud infra).",
+    )
+    trivy_images: list[str] = Field(default_factory=list)
+    iac_directories: list[str] = Field(default_factory=list)
+    timeout_seconds: int = Field(default=3600, ge=60)
+
+
+class GraphConfig(BaseModel):
+    """Graph engine configuration."""
+
+    persist_graphml: bool = Field(default=True)
+    max_nodes_warn: int = Field(default=10000)
+    compute_attack_paths: bool = Field(default=True)
+    export_cytoscape: bool = Field(default=False)
+
+
+class OntologyConfig(BaseModel):
+    """Ontology engine configuration."""
+
+    enabled: bool = Field(default=True)
+    export_formats: list[str] = Field(default=["turtle", "json-ld"])
+    include_raw_metadata: bool = Field(default=False)
+
+
+class RAGConfig(BaseModel):
+    """RAG export configuration."""
+
+    enabled: bool = Field(default=True)
+    chunk_strategy: str = Field(
+        default="hybrid",
+        pattern="^(entity|community|relation_group|hybrid)$",
+        description="Chunking strategy: entity, community, relation_group, or hybrid (all)",
+    )
+    max_chunk_tokens: int = Field(default=2000, ge=100)
+
+
+class TerraformConfig(BaseModel):
+    """Terraform export configuration."""
+
+    enabled: bool = Field(default=False)
+    output_dir: str = Field(default="./reports/terraform")
+
+
+class ReportConfig(BaseModel):
+    """Report generation configuration."""
+
+    formats: list[str] = Field(default=["html", "json", "svg"])
+    inline_js: bool = Field(
+        default=True, description="Inline JS libraries for air-gapped environments"
+    )
+    output_dir: str = Field(default="./reports")
+
+
+def _default_rules_dir() -> str:
+    """Packaged rules directory, falling back to ./rules for old layouts."""
+    packaged = Path(__file__).parent / "rules"
+    if packaged.is_dir():
+        return str(packaged)
+    return "./rules"
+
+
+class RulesetConfig(BaseModel):
+    """External ruleset configuration."""
+
+    rules_dir: str = Field(
+        default_factory=_default_rules_dir,
+        description="Directory containing YAML rulesets (defaults to the rulesets shipped in the package)",
+    )
+    load_external: bool = Field(default=True)
+
+
+class CloudGConfig(BaseModel):
+    """Root configuration model."""
+
+    # Multi-provider support: list of providers to scan simultaneously
+    providers: list[str] = Field(
+        default=["aws"],
+        description="Providers to scan: aws, azure, gcp. Use multiple for simultaneous scanning.",
+    )
+    # Backward-compat alias — single provider string is auto-wrapped
+    provider: str | None = Field(
+        default=None,
+        exclude=True,
+        description="Deprecated: use 'providers' list. Kept for backward compat.",
+    )
+    aws: AWSConfig = Field(default_factory=AWSConfig)
+    azure: AzureConfig = Field(default_factory=AzureConfig)
+    gcp: GCPConfig = Field(default_factory=GCPConfig)
+    scanners: ScannerConfig = Field(default_factory=ScannerConfig)
+    graph: GraphConfig = Field(default_factory=GraphConfig)
+    ontology: OntologyConfig = Field(default_factory=OntologyConfig)
+    rag: RAGConfig = Field(default_factory=RAGConfig)
+    terraform: TerraformConfig = Field(default_factory=TerraformConfig)
+    report: ReportConfig = Field(default_factory=ReportConfig)
+    rulesets: RulesetConfig = Field(default_factory=RulesetConfig)
+    log_file: str | None = None
+    verbose: bool = False
+    concurrency_limit: int = Field(default=5, ge=1, le=50, description="Max concurrent API calls")
+
+    def model_post_init(self, __context: Any) -> None:
+        """Handle backward-compat: if 'provider' is set but 'providers' is default, migrate."""
+        if self.provider is not None and self.providers == ["aws"]:
+            self.providers = [self.provider]
+
+
+def load_config(config_path: str | Path | None = None) -> CloudGConfig:
+    """Load configuration from YAML file with Pydantic validation.
+
+    Args:
+        config_path: Path to config.yaml. If None, uses defaults.
+
+    Returns:
+        Validated CloudGConfig instance.
+    """
+    if config_path is None:
+        logger.info("No config file specified, using defaults")
+        return CloudGConfig()
+
+    path = Path(config_path)
+    if not path.exists():
+        logger.warning("Config file %s not found, using defaults", path)
+        return CloudGConfig()
+
+    try:
+        import yaml
+    except ImportError:
+        logger.warning("PyYAML not installed, using defaults. Install with: pip install pyyaml")
+        return CloudGConfig()
+
+    with open(path) as f:
+        data: dict[str, Any] = yaml.safe_load(f) or {}
+
+    config = CloudGConfig.model_validate(data)
+    logger.info("Loaded config from %s", path)
+    return config

cloudg/coverage.py

@@ -0,0 +1,93 @@
+"""Collection coverage tracking — records which services succeeded/failed."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class ServiceStatus(str, Enum):
+    """Status of a service collection attempt."""
+
+    SUCCESS = "SUCCESS"
+    FAILED = "FAILED"
+    PARTIAL = "PARTIAL"
+    SKIPPED = "SKIPPED"
+
+
+class ServiceCoverage(BaseModel):
+    """Coverage record for a single service."""
+
+    service: str
+    status: ServiceStatus
+    asset_count: int = 0
+    error: str | None = None
+    duration_ms: int | None = None
+
+
+class CollectionCoverage(BaseModel):
+    """Aggregate coverage for an entire collection run."""
+
+    provider: str
+    region: str | None = None
+    account_id: str | None = None
+    started_at: datetime = Field(default_factory=datetime.utcnow)
+    completed_at: datetime | None = None
+    services: list[ServiceCoverage] = Field(default_factory=list)
+
+    @property
+    def total_services(self) -> int:
+        return len(self.services)
+
+    @property
+    def successful_services(self) -> int:
+        return sum(1 for s in self.services if s.status == ServiceStatus.SUCCESS)
+
+    @property
+    def failed_services(self) -> int:
+        return sum(1 for s in self.services if s.status == ServiceStatus.FAILED)
+
+    @property
+    def coverage_pct(self) -> float:
+        if not self.services:
+            return 0.0
+        return round(self.successful_services / self.total_services * 100, 1)
+
+    def record(
+        self,
+        service: str,
+        status: ServiceStatus,
+        asset_count: int = 0,
+        error: str | None = None,
+        duration_ms: int | None = None,
+    ) -> None:
+        """Record a service collection result."""
+        self.services.append(
+            ServiceCoverage(
+                service=service,
+                status=status,
+                asset_count=asset_count,
+                error=error,
+                duration_ms=duration_ms,
+            )
+        )
+
+    def to_summary(self) -> dict[str, Any]:
+        """Return a summary dict for reporting."""
+        return {
+            "provider": self.provider,
+            "region": self.region,
+            "account_id": self.account_id,
+            "total_services": self.total_services,
+            "successful": self.successful_services,
+            "failed": self.failed_services,
+            "coverage_pct": self.coverage_pct,
+            "failures": [
+                {"service": s.service, "error": s.error}
+                for s in self.services
+                if s.status == ServiceStatus.FAILED
+            ],
+        }