argus-cloud-optimizer 0.2.0__py3-none-any.whl

adapters/azure/cost_management.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import structlog
+from azure.core.exceptions import HttpResponseError
+from azure.identity import DefaultAzureCredential
+from azure.mgmt.costmanagement import CostManagementClient
+from azure.mgmt.costmanagement.models import (
+    QueryComparisonExpression,
+    QueryDataset,
+    QueryDefinition,
+    QueryFilter,
+    QueryGrouping,
+    QueryTimePeriod,
+)
+
+from adapters.azure.retry import retry_on_transient
+
+logger = structlog.get_logger(__name__)
+
+_BATCH_SIZE = 50  # Cost Management API supports up to ~100 resource IDs per filter
+
+
+def get_cost(
+    subscription_id: str,
+    resource_ids: list[str],
+    days: int = 30,
+    credential: Any = None,
+) -> dict[str, float]:
+    """
+    Return estimated cost in USD per resource ID over the last N days.
+
+    Uses Azure Cost Management QueryUsage API, batched to avoid filter size limits.
+    Cost Management requires the subscription to have a spending plan (not free tier).
+    Returns zeros with a warning if cost data is unavailable.
+    """
+    if not resource_ids:
+        return {}
+
+    cred = credential or DefaultAzureCredential()
+    client = CostManagementClient(cred, connection_timeout=10, read_timeout=60)
+    scope = f"/subscriptions/{subscription_id}"
+
+    costs: dict[str, float] = {rid: 0.0 for rid in resource_ids}
+
+    # Process in batches to stay within API filter limits
+    for i in range(0, len(resource_ids), _BATCH_SIZE):
+        batch = resource_ids[i : i + _BATCH_SIZE]
+        try:
+            _query_batch(client, scope, batch, days, costs)
+        except HttpResponseError as exc:
+            if exc.status_code in (403, 404):
+                logger.warning(
+                    "azure_cost_management_unavailable",
+                    extra={
+                        "subscription_id": subscription_id,
+                        "error": str(exc),
+                        "hint": (
+                            "Cost Management requires a paid Azure subscription. "
+                            "Free trial accounts return no cost data."
+                        ),
+                    },
+                )
+                break
+            logger.error(
+                "azure_cost_management_failed",
+                extra={"subscription_id": subscription_id, "error": str(exc)},
+            )
+
+    logger.info(
+        "azure_cost_query_complete",
+        extra={
+            "subscription_id": subscription_id,
+            "resources_queried": len(resource_ids),
+            "resources_with_cost": sum(1 for v in costs.values() if v > 0),
+        },
+    )
+    return costs
+
+
+def _query_batch(
+    client: CostManagementClient,
+    scope: str,
+    resource_ids: list[str],
+    days: int,
+    costs: dict[str, float],
+) -> None:
+    end_date = datetime.now(tz=timezone.utc)
+    start_date = end_date - timedelta(days=days)
+
+    query = QueryDefinition(
+        type="Usage",
+        timeframe="Custom",
+        time_period=QueryTimePeriod(
+            from_property=start_date,
+            to=end_date,
+        ),
+        dataset=QueryDataset(
+            granularity="None",
+            aggregation={"totalCost": {"name": "PreTaxCost", "function": "Sum"}},  # type: ignore[dict-item]
+            grouping=[QueryGrouping(type="Dimension", name="ResourceId")],
+            filter=QueryFilter(
+                dimensions=QueryComparisonExpression(
+                    name="ResourceId",
+                    operator="In",
+                    values=resource_ids,
+                )
+            ),
+        ),
+    )
+
+    result = retry_on_transient(client.query.usage, scope=scope, parameters=query)
+
+    # Result rows: [cost, currency, resourceId]
+    for row in result.rows if result and result.rows else []:
+        if len(row) >= 3:
+            amount = float(row[0])
+            resource_id: str = str(row[2])
+            # Match case-insensitively — Azure resource IDs are case-insensitive
+            for rid in costs:
+                if rid.lower() == resource_id.lower():
+                    costs[rid] = costs[rid] + amount
+                    break

adapters/azure/monitor.py

@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import structlog
+from azure.core.exceptions import HttpResponseError
+from azure.identity import DefaultAzureCredential
+from azure.monitor.query import MetricAggregationType, MetricsQueryClient
+
+from adapters.azure.retry import retry_on_transient
+from adapters.base import MetricSummary
+
+logger = structlog.get_logger(__name__)
+
+# (MetricName, AggregationType)
+_METRICS: dict[str, list[tuple[str, str]]] = {
+    # Virtual Machines
+    "microsoft.compute/virtualmachines": [
+        ("Percentage CPU", "Average"),
+        ("Network In Total", "Total"),
+        ("Network Out Total", "Total"),
+    ],
+    # VM Scale Sets
+    "microsoft.compute/virtualmachinescalesets": [
+        ("Percentage CPU", "Average"),
+        ("Network In Total", "Total"),
+        ("Network Out Total", "Total"),
+    ],
+    # Managed Disks
+    "microsoft.compute/disks": [
+        ("Composite Disk Read Operations/sec", "Average"),
+        ("Composite Disk Write Operations/sec", "Average"),
+    ],
+    # Azure SQL Database
+    "microsoft.sql/servers/databases": [
+        ("cpu_percent", "Average"),
+        ("connection_successful", "Total"),
+        ("storage_percent", "Average"),
+    ],
+    # Azure SQL Managed Instance
+    "microsoft.sql/managedinstances": [
+        ("avg_cpu_percent", "Average"),
+        ("storage_space_used_mb", "Average"),
+    ],
+    # App Service Plans
+    "microsoft.web/serverfarms": [
+        ("CpuPercentage", "Average"),
+        ("MemoryPercentage", "Average"),
+        ("HttpQueueLength", "Average"),
+    ],
+    # App Services / Function Apps
+    "microsoft.web/sites": [
+        ("CpuTime", "Total"),
+        ("Requests", "Total"),
+        ("BytesReceived", "Total"),
+    ],
+    # AKS Clusters
+    "microsoft.containerservice/managedclusters": [
+        ("node_cpu_usage_percentage", "Average"),
+        ("node_memory_rss_percentage", "Average"),
+        ("kube_node_status_allocatable_cpu_cores", "Average"),
+    ],
+    # Container Instances
+    "microsoft.containerinstance/containergroups": [
+        ("CpuUsage", "Average"),
+        ("MemoryUsage", "Average"),
+        ("NetworkBytesReceivedPerSecond", "Average"),
+    ],
+    # Azure Cache for Redis
+    "microsoft.cache/redis": [
+        ("connectedclients", "Average"),
+        ("cachehits", "Total"),
+        ("cachemisses", "Total"),
+    ],
+    # Cosmos DB
+    "microsoft.documentdb/databaseaccounts": [
+        ("TotalRequests", "Total"),
+        ("NormalizedRUConsumption", "Average"),
+        ("ServerSideLatency", "Average"),
+    ],
+    # Storage Accounts
+    "microsoft.storage/storageaccounts": [
+        ("Transactions", "Total"),
+        ("Ingress", "Total"),
+        ("Egress", "Total"),
+    ],
+    # Azure Kubernetes Service Node Pools
+    "microsoft.containerservice/managedclusters/agentpools": [
+        ("node_cpu_usage_percentage", "Average"),
+        ("node_memory_rss_percentage", "Average"),
+    ],
+    # Event Hubs
+    "microsoft.eventhub/namespaces": [
+        ("IncomingMessages", "Total"),
+        ("OutgoingMessages", "Total"),
+        ("ActiveConnections", "Average"),
+    ],
+    # Service Bus
+    "microsoft.servicebus/namespaces": [
+        ("IncomingMessages", "Total"),
+        ("OutgoingMessages", "Total"),
+        ("ActiveConnections", "Average"),
+    ],
+    # Azure Functions (same as web/sites but grouped for clarity)
+    "microsoft.web/sites/functions": [
+        ("FunctionExecutionCount", "Total"),
+        ("FunctionExecutionUnits", "Total"),
+    ],
+    # API Management
+    "microsoft.apimanagement/service": [
+        ("TotalRequests", "Total"),
+        ("SuccessfulRequests", "Total"),
+        ("Capacity", "Average"),
+    ],
+    # Application Gateway
+    "microsoft.network/applicationgateways": [
+        ("TotalRequests", "Total"),
+        ("CurrentConnections", "Average"),
+        ("Throughput", "Average"),
+    ],
+    # Load Balancers
+    "microsoft.network/loadbalancers": [
+        ("PacketCount", "Total"),
+        ("ByteCount", "Total"),
+        ("AllocatedSnatPorts", "Average"),
+    ],
+    # Azure Databricks
+    "microsoft.databricks/workspaces": [
+        ("autoOptimizeClusterUtilization", "Average"),
+        ("numActiveClusters", "Average"),
+    ],
+    # HDInsight
+    "microsoft.hdinsight/clusters": [
+        ("GatewayRequests", "Total"),
+        ("CategorizedGatewayRequests", "Total"),
+    ],
+    # Logic Apps
+    "microsoft.logic/workflows": [
+        ("RunsStarted", "Total"),
+        ("RunsCompleted", "Total"),
+        ("RunsFailed", "Total"),
+    ],
+    # Cognitive Services / OpenAI
+    "microsoft.cognitiveservices/accounts": [
+        ("TotalCalls", "Total"),
+        ("TotalErrors", "Total"),
+        ("Latency", "Average"),
+    ],
+    # Azure Search
+    "microsoft.search/searchservices": [
+        ("SearchQueriesPerSecond", "Average"),
+        ("ThrottledSearchQueriesPercentage", "Average"),
+    ],
+    # Azure Stream Analytics
+    "microsoft.streamanalytics/streamingjobs": [
+        ("InputEvents", "Total"),
+        ("OutputEvents", "Total"),
+        ("ResourceUtilization", "Average"),
+    ],
+    # Azure Data Factory
+    "microsoft.datafactory/factories": [
+        ("PipelineRunsStarted", "Total"),
+        ("ActivityRunsStarted", "Total"),
+        ("TriggerRunsStarted", "Total"),
+    ],
+}
+
+_AGGREGATION_MAP = {
+    "Average": MetricAggregationType.AVERAGE,
+    "Total": MetricAggregationType.TOTAL,
+    "Minimum": MetricAggregationType.MINIMUM,
+    "Maximum": MetricAggregationType.MAXIMUM,
+}
+
+_FALLBACK_METRIC_LIMIT = 5
+
+
+def get_metrics(
+    resource_id: str,
+    resource_type: str,
+    days: int = 90,
+    credential: Any = None,
+) -> MetricSummary:
+    """
+    Fetch Azure Monitor metrics for a resource.
+    Falls back to querying available metric definitions for unknown resource types.
+    """
+    metric_defs = _METRICS.get(resource_type.lower())
+    if not metric_defs:
+        metric_defs = _discover_metrics(resource_id, resource_type, credential)
+    if not metric_defs:
+        return MetricSummary(
+            resource_id=resource_id,
+            resource_type=resource_type,
+            period_days=days,
+            metrics={},
+            has_data=False,
+        )
+
+    cred = credential or DefaultAzureCredential()
+    client = MetricsQueryClient(cred, connection_timeout=10, read_timeout=60)
+
+    end_time = datetime.now(tz=timezone.utc)
+    start_time = end_time - timedelta(days=days)
+    granularity = timedelta(days=1)
+
+    metric_names = [name for name, _ in metric_defs]
+
+    try:
+        response = retry_on_transient(
+            client.query_resource,
+            resource_uri=resource_id,
+            metric_names=metric_names,
+            timespan=(start_time, end_time),
+            granularity=granularity,
+        )
+    except HttpResponseError as exc:
+        logger.warning(
+            "azure_monitor_query_failed",
+            extra={"resource_id": resource_id, "error": str(exc)},
+        )
+        return MetricSummary(
+            resource_id=resource_id,
+            resource_type=resource_type,
+            period_days=days,
+            metrics={},
+            has_data=False,
+        )
+
+    return _parse_response(response, metric_defs, resource_id, resource_type, days)
+
+
+def _parse_response(
+    response: Any,
+    metric_defs: list[tuple[str, str]],
+    resource_id: str,
+    resource_type: str,
+    days: int,
+) -> MetricSummary:
+    metrics: dict[str, Any] = {}
+    has_data = False
+
+    agg_map = {name: agg for name, agg in metric_defs}
+
+    for metric in response.metrics:
+        name: str = metric.name
+        agg_type: str = agg_map.get(name, "Average")
+        values: list[float] = []
+
+        for ts in metric.timeseries:
+            for data_point in ts.data:
+                val = data_point.total if agg_type == "Total" else data_point.average
+                if val is not None:
+                    values.append(val)
+
+        if not values:
+            metrics[name] = None
+            continue
+
+        has_data = True
+        metrics[name] = round(
+            sum(values) if agg_type == "Total" else sum(values) / len(values), 4
+        )
+
+    return MetricSummary(
+        resource_id=resource_id,
+        resource_type=resource_type,
+        period_days=days,
+        metrics=metrics,
+        has_data=has_data,
+    )
+
+
+def _discover_metrics(
+    resource_id: str,
+    resource_type: str,
+    credential: Any = None,
+) -> list[tuple[str, str]]:
+    """Auto-discover available metrics for unknown Azure resource types."""
+    from azure.monitor.query import MetricsQueryClient
+
+    cred = credential or DefaultAzureCredential()
+    client = MetricsQueryClient(cred, connection_timeout=10, read_timeout=60)
+    discovered: list[tuple[str, str]] = []
+
+    try:
+        definitions = retry_on_transient(
+            client.list_metric_definitions, resource_uri=resource_id
+        )
+        for defn in definitions:
+            metric_name: str = defn.name or ""
+            # Pick aggregation based on metric name heuristics
+            agg = (
+                "Total"
+                if any(
+                    kw in metric_name.lower()
+                    for kw in ("count", "bytes", "requests", "transactions", "total")
+                )
+                else "Average"
+            )
+            discovered.append((metric_name, agg))
+            if len(discovered) >= _FALLBACK_METRIC_LIMIT:
+                break
+    except HttpResponseError as exc:
+        logger.warning(
+            "azure_monitor_list_metrics_failed",
+            extra={"resource_id": resource_id, "error": str(exc)},
+        )
+
+    return discovered

adapters/azure/resource_graph.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from typing import Any
+
+import structlog
+from azure.core.exceptions import HttpResponseError
+from azure.identity import DefaultAzureCredential
+from azure.mgmt.resourcegraph import ResourceGraphClient
+from azure.mgmt.resourcegraph.models import QueryRequest, QueryRequestOptions
+
+from adapters.azure.retry import retry_on_transient
+from adapters.base import Resource
+
+logger = structlog.get_logger(__name__)
+
+# KQL query — returns all resources with their type, location, tags, and resource group.
+# We exclude resource types that have no billing impact (e.g. locks, role assignments).
+_RESOURCE_QUERY = """
+Resources
+| where type !in~ (
+    'microsoft.authorization/roleassignments',
+    'microsoft.authorization/roledefinitions',
+    'microsoft.authorization/locks',
+    'microsoft.resources/deployments',
+    'microsoft.resources/tags'
+)
+| project id, name, type, location, resourceGroup, tags, subscriptionId
+"""
+
+_PAGE_SIZE = 1000  # Resource Graph max per page
+
+
+def list_resources(
+    subscription_ids: list[str],
+    ignore_regions: list[str] | None = None,
+    credential: Any = None,
+) -> list[Resource]:
+    """
+    Return all billable Azure resources across the given subscriptions
+    using Azure Resource Graph (single cross-subscription query).
+
+    Auth: DefaultAzureCredential — Managed Identity in production,
+    az login / env vars for local dev.
+    """
+    cred = credential or DefaultAzureCredential()
+    client = ResourceGraphClient(cred, connection_timeout=10, read_timeout=60)
+    ignore_set = {r.lower() for r in (ignore_regions or [])}
+    resources: list[Resource] = []
+
+    request = QueryRequest(
+        subscriptions=subscription_ids,
+        query=_RESOURCE_QUERY,
+        options=QueryRequestOptions(result_format="objectArray", top=_PAGE_SIZE),
+    )
+
+    skip_token: str | None = None
+
+    try:
+        while True:
+            if skip_token:
+                request.options.skip_token = skip_token
+
+            response = retry_on_transient(client.resources, request)
+
+            for raw in response.data or []:
+                parsed = _parse_resource(raw, ignore_set)
+                if parsed:
+                    resources.append(parsed)
+
+            _raw_token = getattr(response, "skip_token", None) or getattr(
+                response, "$skipToken", None
+            )
+            skip_token = _raw_token if isinstance(_raw_token, str) else None
+            if not skip_token:
+                break
+
+    except HttpResponseError as exc:
+        if exc.status_code == 403:
+            raise PermissionError(
+                "Argus service principal is missing Reader role "
+                "on the subscription(s). "
+                "Assign 'Reader' at the subscription scope."
+            ) from exc
+        raise
+
+    logger.info(
+        "resource_graph_query_complete",
+        extra={"subscriptions": subscription_ids, "total": len(resources)},
+    )
+    return resources
+
+
+def _parse_resource(raw: dict[str, Any], ignore_set: set[str]) -> Resource | None:
+    resource_id: str = raw.get("id", "")
+    name: str = raw.get("name", "")
+    resource_type: str = raw.get("type", "")
+    location: str = raw.get("location", "global")
+
+    if not resource_id or not resource_type:
+        return None
+    if location.lower() in ignore_set:
+        return None
+
+    tags: dict[str, str] = {str(k): str(v) for k, v in (raw.get("tags") or {}).items()}
+
+    return Resource(
+        resource_id=resource_id,
+        resource_type=resource_type.lower(),
+        cloud="azure",
+        region=location,
+        name=name or None,
+        tags=tags,
+    )

adapters/azure/retry.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import random
+import time
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+import structlog
+from azure.core.exceptions import HttpResponseError
+
+logger = structlog.get_logger(__name__)
+
+T = TypeVar("T")
+
+_MAX_RETRIES = 3
+_BASE_DELAY = 1.0
+
+
+def retry_on_transient(
+    fn: Callable[..., T],
+    *args: Any,
+    **kwargs: Any,
+) -> T:
+    delay = _BASE_DELAY
+    for attempt in range(_MAX_RETRIES):
+        try:
+            return fn(*args, **kwargs)
+        except HttpResponseError as exc:
+            status = exc.status_code or 0
+            if status in (429, 500, 502, 503, 504) and attempt < _MAX_RETRIES - 1:
+                retry_after = _parse_retry_after(exc)
+                jitter = random.uniform(0, delay * 0.5)  # noqa: S311
+                sleep_time = retry_after if retry_after else delay + jitter
+                logger.warning(
+                    "azure_transient_error_retrying",
+                    status_code=status,
+                    attempt=attempt + 1,
+                    max_retries=_MAX_RETRIES,
+                    retry_in=round(sleep_time, 1),
+                )
+                time.sleep(sleep_time)
+                delay *= 2
+            else:
+                raise
+    raise RuntimeError("Unreachable")  # pragma: no cover
+
+
+def _parse_retry_after(exc: HttpResponseError) -> float | None:
+    if exc.response is None:
+        return None
+    header = exc.response.headers.get("Retry-After")
+    if header is None:
+        return None
+    try:
+        return float(header)
+    except ValueError:
+        return None