argus-cloud-optimizer 0.2.0__py3-none-any.whl

adapters/base.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class Resource:
+    """Minimal representation of a discovered cloud resource."""
+
+    resource_id: str
+    resource_type: str  # e.g. "AWS::EC2::Instance"
+    cloud: str  # "aws" | "gcp" | "azure"
+    region: str
+    name: str | None = None
+    tags: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "resource_id": self.resource_id,
+            "resource_type": self.resource_type,
+            "cloud": self.cloud,
+            "region": self.region,
+            "name": self.name,
+            "tags": self.tags,
+        }
+
+
+@dataclass
+class MetricSummary:
+    """Key usage metrics for a resource over a lookback window."""
+
+    resource_id: str
+    resource_type: str
+    period_days: int
+    metrics: dict[str, Any]  # {"avg_cpu_pct": 1.2, "network_bytes_total": 847, ...}
+    has_data: bool = True  # False if CloudWatch has no data points
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "resource_id": self.resource_id,
+            "resource_type": self.resource_type,
+            "period_days": self.period_days,
+            "metrics": self.metrics,
+            "has_data": self.has_data,
+        }
+
+
+class CloudAdapter(ABC):
+    """
+    Abstract cloud adapter. One implementation per cloud provider.
+    The agent loop only ever calls these four methods — never raw SDK clients.
+    All implementations must be read-only (no mutations to cloud resources).
+    """
+
+    @abstractmethod
+    def list_resources(self, ignore_regions: list[str] | None = None) -> list[Resource]:
+        """
+        Return every resource across ALL regions, excluding ignore_regions.
+        Empty or None means scan everything — new regions are included automatically.
+        Implementation uses Resource Explorer (AWS), Asset Inventory (GCP),
+        or Resource Graph (Azure). Never hardcode resource types.
+        """
+        ...
+
+    @abstractmethod
+    def get_metrics(
+        self,
+        resource_id: str,
+        resource_type: str,
+        days: int = 90,
+    ) -> MetricSummary:
+        """
+        Fetch usage metrics relevant to this resource type over the last N days.
+        The adapter decides which metrics matter per resource type.
+        Default is 90 days — covers quarterly usage patterns. Override via
+        METRICS_LOOKBACK_DAYS env var (see cloudwatch.DEFAULT_METRICS_DAYS).
+        """
+        ...
+
+    @abstractmethod
+    def get_cost(
+        self,
+        resource_ids: list[str],
+        days: int = 30,
+    ) -> dict[str, float]:
+        """
+        Return estimated monthly cost in USD per resource ID.
+        Always batch resource_ids — never call per-resource.
+        """
+        ...
+
+    @abstractmethod
+    def get_last_activity(
+        self,
+        resource_id: str,
+        resource_type: str,
+    ) -> datetime | None:
+        """
+        Return the timestamp of the last meaningful activity for this resource.
+        Returns None if no activity found in the lookback window.
+        """
+        ...

adapters/gcp/adapter.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import os
+from datetime import datetime
+
+from adapters.base import CloudAdapter, MetricSummary, Resource
+from adapters.gcp import asset_inventory, billing, cloud_logging, cloud_monitoring
+
+
+class GCPAdapter(CloudAdapter):
+    """
+    GCP implementation of CloudAdapter.
+    Wires together Cloud Asset Inventory, Cloud Monitoring, Billing (BigQuery),
+    and Cloud Audit Logs. All API calls are read-only.
+
+    Auth: uses Application Default Credentials (ADC).
+    - Cloud Run Job: the service account attached to the job
+    - Local dev: `gcloud auth application-default login`
+
+    Usage:
+        adapter = GCPAdapter(project_id="my-gcp-project")
+    """
+
+    def __init__(
+        self,
+        project_id: str | None = None,
+        bq_billing_table: str | None = None,
+    ) -> None:
+        resolved = project_id or os.environ.get("GCP_PROJECT_ID", "")
+        if not resolved:
+            raise EnvironmentError(
+                "GCP_PROJECT_ID is not set. "
+                "Pass project_id= or export GCP_PROJECT_ID."
+            )
+        self._project_id: str = resolved
+        self._bq_billing_table = bq_billing_table or os.environ.get("BILLING_BQ_TABLE")
+
+    def list_resources(self, ignore_regions: list[str] | None = None) -> list[Resource]:
+        return asset_inventory.list_resources(
+            project_id=self._project_id,
+            ignore_regions=ignore_regions,
+        )
+
+    def get_metrics(
+        self,
+        resource_id: str,
+        resource_type: str,
+        days: int = 90,
+    ) -> MetricSummary:
+        return cloud_monitoring.get_metrics(
+            project_id=self._project_id,
+            resource_id=resource_id,
+            resource_type=resource_type,
+            days=days,
+        )
+
+    def get_cost(
+        self,
+        resource_ids: list[str],
+        days: int = 30,
+    ) -> dict[str, float]:
+        return billing.get_cost(
+            project_id=self._project_id,
+            resource_ids=resource_ids,
+            days=days,
+            bq_table=self._bq_billing_table,
+        )
+
+    def get_last_activity(
+        self,
+        resource_id: str,
+        resource_type: str,
+    ) -> datetime | None:
+        return cloud_logging.get_last_activity(
+            project_id=self._project_id,
+            resource_id=resource_id,
+            resource_type=resource_type,
+        )
+
+    @classmethod
+    def from_env(cls) -> "GCPAdapter":
+        """Convenience constructor — reads all config from env vars."""
+        return cls(
+            project_id=os.environ.get("GCP_PROJECT_ID"),
+            bq_billing_table=os.environ.get("BILLING_BQ_TABLE"),
+        )

adapters/gcp/asset_inventory.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+from google.api_core.exceptions import GoogleAPICallError, PermissionDenied
+from google.cloud import asset_v1
+
+from adapters.base import Resource
+from adapters.gcp.retry import retry_on_transient
+
+logger = structlog.get_logger(__name__)
+
+# Asset types Argus cares about. Empty list = all types (too noisy for cost analysis).
+# We scope to resource types that have associated billing.
+SCANNED_ASSET_TYPES: list[str] = [
+    "compute.googleapis.com/Instance",
+    "compute.googleapis.com/Disk",
+    "compute.googleapis.com/Address",  # static IPs
+    "compute.googleapis.com/ForwardingRule",
+    "compute.googleapis.com/BackendService",
+    "sql.googleapis.com/Instance",  # Cloud SQL
+    "container.googleapis.com/Cluster",  # GKE
+    "run.googleapis.com/Service",  # Cloud Run
+    "cloudfunctions.googleapis.com/Function",  # Cloud Functions
+    "storage.googleapis.com/Bucket",
+    "bigquery.googleapis.com/Dataset",
+    "bigquery.googleapis.com/Table",
+    "redis.googleapis.com/Instance",  # Memorystore Redis
+    "spanner.googleapis.com/Instance",
+    "bigtable.googleapis.com/Instance",
+    "pubsub.googleapis.com/Topic",
+    "pubsub.googleapis.com/Subscription",
+    "dataflow.googleapis.com/Job",
+    "dataproc.googleapis.com/Cluster",
+    "aiplatform.googleapis.com/Endpoint",  # Vertex AI
+    "composer.googleapis.com/Environment",  # Cloud Composer (Airflow)
+    "notebooks.googleapis.com/Instance",  # Vertex AI Workbench
+]
+
+
+def list_resources(
+    project_id: str,
+    ignore_regions: list[str] | None = None,
+) -> list[Resource]:
+    """
+    Return all billable GCP resources in a project using Cloud Asset Inventory.
+    Uses a single paginated API call — no per-resource-type enumeration needed.
+    """
+    client = asset_v1.AssetServiceClient()
+    parent = f"projects/{project_id}"
+    ignore_set = set(ignore_regions or [])
+    resources: list[Resource] = []
+
+    request = asset_v1.ListAssetsRequest(
+        parent=parent,
+        asset_types=SCANNED_ASSET_TYPES,
+        content_type=asset_v1.ContentType.RESOURCE,
+    )
+
+    try:
+        for asset in retry_on_transient(
+            client.list_assets, request=request, timeout=60
+        ):
+            parsed = _parse_asset(asset, ignore_set)
+            if parsed:
+                resources.append(parsed)
+    except PermissionDenied as exc:
+        raise PermissionError(
+            f"Argus service account is missing cloudasset.assets.listAssets "
+            f"permission on project {project_id}."
+        ) from exc
+    except GoogleAPICallError as exc:
+        raise RuntimeError(f"Cloud Asset Inventory API error: {exc}") from exc
+
+    logger.info(
+        "asset_inventory_complete",
+        extra={"project_id": project_id, "total": len(resources)},
+    )
+    return resources
+
+
+def _parse_asset(asset: Any, ignore_set: set[str]) -> Resource | None:
+    resource = asset.resource
+    if not resource:
+        return None
+
+    data: dict[str, Any] = dict(resource.data)
+    name: str = asset.name  # full resource name: //compute.googleapis.com/projects/…
+    asset_type: str = asset.asset_type  # e.g. compute.googleapis.com/Instance
+    location: str = data.get("location", data.get("zone", data.get("region", "global")))
+
+    # Normalise zone (us-central1-a) to region (us-central1)
+    region = _to_region(location)
+    if region in ignore_set:
+        return None
+
+    labels: dict[str, str] = dict(data.get("labels", {}))
+    friendly_name: str | None = data.get("name") or data.get("displayName")
+
+    return Resource(
+        resource_id=name,
+        resource_type=asset_type,
+        cloud="gcp",
+        region=region,
+        name=friendly_name,
+        tags=labels,
+    )
+
+
+def _to_region(location: str) -> str:
+    """Strip the zone suffix from a zone string to get the region."""
+    parts = location.rsplit("-", 1)
+    if len(parts) == 2 and len(parts[1]) == 1 and parts[1].isalpha():
+        return parts[0]
+    return location

adapters/gcp/billing.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+# BigQuery dataset where Cloud Billing export is written.
+# Users must enable billing export to BigQuery — this is the standard GCP cost path.
+# Set via BILLING_BQ_DATASET env var: "project.dataset" or "project.dataset.table"
+_DEFAULT_TABLE = "argus_billing.gcp_billing_export_v1"
+
+
+def get_cost(
+    project_id: str,
+    resource_ids: list[str],
+    days: int = 30,
+    bq_table: str | None = None,
+) -> dict[str, float]:
+    """
+    Return estimated cost in USD per resource ID over the last N days.
+
+    GCP billing data is available via two paths:
+    1. Cloud Billing Budget API — account-level budgets only, no per-resource breakdown.
+    2. BigQuery billing export — per-resource cost, requires export to be enabled.
+
+    We use the BigQuery export path since it's the only way to get per-resource cost.
+    If the export table doesn't exist or isn't configured, returns zeros with a warning.
+
+    The caller is responsible for passing resource_ids as the full GCP resource names
+    (//compute.googleapis.com/projects/…) — we extract the short name for BQ filtering.
+    """
+    if not resource_ids:
+        return {}
+
+    import os
+
+    resolved_table = bq_table or os.environ.get("BILLING_BQ_TABLE", _DEFAULT_TABLE)
+
+    try:
+        return _query_bigquery(project_id, resource_ids, days, resolved_table or "")
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "gcp_billing_query_failed",
+            extra={
+                "project_id": project_id,
+                "error": str(exc),
+                "hint": (
+                    "Enable Cloud Billing export to BigQuery in the GCP console "
+                    "(Billing → Billing export → BigQuery export). "
+                    "Set BILLING_BQ_TABLE env var to 'project.dataset.table'."
+                ),
+            },
+        )
+        return {rid: 0.0 for rid in resource_ids}
+
+
+def _query_bigquery(
+    project_id: str,
+    resource_ids: list[str],
+    days: int,
+    bq_table: str,
+) -> dict[str, float]:
+    from google.cloud import bigquery  # type: ignore[import-untyped,attr-defined]
+
+    client = bigquery.Client(project=project_id)
+    end_date = datetime.now(tz=timezone.utc).date()
+    start_date = end_date - timedelta(days=days)
+
+    # Extract short resource names from full asset names for matching.
+    # Full: //compute.googleapis.com/projects/p/zones/z/instances/my-vm
+    # Short: my-vm
+    short_names = [rid.rstrip("/").split("/")[-1] for rid in resource_ids]
+    name_to_full = {rid.rstrip("/").split("/")[-1]: rid for rid in resource_ids}
+
+    placeholders = ", ".join(f"@name_{i}" for i in range(len(short_names)))
+    query = f"""
+        SELECT
+            resource.name AS resource_name,
+            SUM(cost) AS total_cost
+        FROM `{bq_table}`
+        WHERE
+            DATE(usage_start_time) >= @start_date
+            AND DATE(usage_end_time) <= @end_date
+            AND resource.name IN ({placeholders})
+        GROUP BY resource.name
+    """
+
+    job_config = bigquery.QueryJobConfig(
+        query_parameters=[
+            bigquery.ScalarQueryParameter("start_date", "DATE", start_date.isoformat()),
+            bigquery.ScalarQueryParameter("end_date", "DATE", end_date.isoformat()),
+            *[
+                bigquery.ScalarQueryParameter(f"name_{i}", "STRING", name)
+                for i, name in enumerate(short_names)
+            ],
+        ]
+    )
+
+    costs: dict[str, float] = {rid: 0.0 for rid in resource_ids}
+    results = client.query(query, job_config=job_config).result()
+
+    for row in results:
+        short = row.resource_name
+        full_id = name_to_full.get(short)
+        if full_id:
+            costs[full_id] = round(float(row.total_cost), 4)
+
+    logger.info(
+        "gcp_billing_query_complete",
+        extra={
+            "project_id": project_id,
+            "resources_queried": len(resource_ids),
+            "resources_with_cost": sum(1 for v in costs.values() if v > 0),
+        },
+    )
+    return costs

adapters/gcp/cloud_logging.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+import structlog
+from google.api_core.exceptions import GoogleAPICallError
+from google.cloud import logging as gcp_logging
+
+from adapters.gcp.retry import retry_on_transient
+
+logger = structlog.get_logger(__name__)
+
+_LOOKBACK_DAYS = 90  # Cloud Logging retention default is 30-400 days depending on tier
+
+
+def get_last_activity(
+    project_id: str,
+    resource_id: str,
+    resource_type: str,
+) -> datetime | None:
+    """
+    Return the timestamp of the most recent admin/data activity for a GCP resource.
+    Uses Cloud Audit Logs (Admin Activity + Data Access) via the Cloud Logging API.
+    Returns None if no activity found in the last 90 days.
+
+    resource_id is a full GCP asset name:
+    //compute.googleapis.com/projects/p/zones/z/instances/my-vm
+    """
+    short_name = resource_id.rstrip("/").split("/")[-1]
+    service = _service_from_resource_type(resource_type)
+
+    client = gcp_logging.Client(project=project_id)
+
+    end_time = datetime.now(tz=timezone.utc)
+    start_time = end_time - timedelta(days=_LOOKBACK_DAYS)
+
+    # Cloud Audit Log filter — matches admin activity on the specific resource.
+    log_filter = (
+        f'logName=("projects/{project_id}/logs/cloudaudit.googleapis.com%2Factivity" '
+        f'OR "projects/{project_id}/logs/cloudaudit.googleapis.com%2Fdata_access") '
+        f'AND resource.labels.resource_name:"{short_name}" '
+        f'AND timestamp >= "{start_time.isoformat()}" '
+        f'AND timestamp <= "{end_time.isoformat()}"'
+    )
+    if service:
+        log_filter += f' AND protoPayload.serviceName="{service}"'
+
+    try:
+        entries = list(
+            retry_on_transient(
+                client.list_entries,
+                filter_=log_filter,
+                order_by=gcp_logging.DESCENDING,
+                page_size=1,
+                timeout=60,
+            )
+        )
+    except GoogleAPICallError as exc:
+        logger.warning(
+            "cloud_logging_lookup_failed",
+            extra={"resource_id": resource_id, "error": str(exc)},
+        )
+        return None
+
+    if not entries:
+        return None
+
+    event_time: datetime = entries[0].timestamp
+    if event_time.tzinfo is None:
+        event_time = event_time.replace(tzinfo=timezone.utc)
+    return event_time
+
+
+def _service_from_resource_type(resource_type: str) -> str | None:
+    """Map GCP asset type to the Cloud Audit Log service name for tighter filtering."""
+    mapping: dict[str, str] = {
+        "compute.googleapis.com/Instance": "compute.googleapis.com",
+        "compute.googleapis.com/Disk": "compute.googleapis.com",
+        "sql.googleapis.com/Instance": "cloudsql.googleapis.com",
+        "container.googleapis.com/Cluster": "container.googleapis.com",
+        "storage.googleapis.com/Bucket": "storage.googleapis.com",
+        "bigquery.googleapis.com/Dataset": "bigquery.googleapis.com",
+        "bigquery.googleapis.com/Table": "bigquery.googleapis.com",
+        "run.googleapis.com/Service": "run.googleapis.com",
+        "cloudfunctions.googleapis.com/Function": "cloudfunctions.googleapis.com",
+        "pubsub.googleapis.com/Topic": "pubsub.googleapis.com",
+        "redis.googleapis.com/Instance": "redis.googleapis.com",
+        "spanner.googleapis.com/Instance": "spanner.googleapis.com",
+        "dataflow.googleapis.com/Job": "dataflow.googleapis.com",
+        "dataproc.googleapis.com/Cluster": "dataproc.googleapis.com",
+        "aiplatform.googleapis.com/Endpoint": "aiplatform.googleapis.com",
+    }
+    return mapping.get(resource_type)