qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

tools/cli_commands/cost_report/cost_management_api.py

@@ -0,0 +1,155 @@
+from collections.abc import Mapping
+from typing import Self
+from urllib.parse import urljoin, urlparse
+
+from requests import Response
+from urllib3.util import Retry
+
+from reconcile.utils.oauth2_backend_application_session import (
+    OAuth2BackendApplicationSession,
+)
+from reconcile.utils.rest_api_base import ApiBase
+from tools.cli_commands.cost_report.response import (
+    AwsReportCostResponse,
+    OpenShiftCostOptimizationReportResponse,
+    OpenShiftReportCostResponse,
+)
+
+REQUEST_TIMEOUT = 60
+PAGE_LIMIT = 100
+MEMORY_UNIT = "MiB"
+CPU_UNIT = "millicores"
+
+
+class CostManagementApi(ApiBase):
+    """
+    Cost Management API client.
+
+    Doc at https://console.redhat.com/docs/api/cost-management
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        token_url: str,
+        client_id: str,
+        client_secret: str,
+        scope: list[str] | None = None,
+    ) -> None:
+        session = OAuth2BackendApplicationSession(
+            client_id=client_id,
+            client_secret=client_secret,
+            token_url=token_url,
+            scope=scope,
+        )
+        max_retries = Retry(
+            total=3,
+            backoff_factor=15,  # large backoff required for server-side processing
+            status_forcelist=[500, 502, 503, 504],
+        )
+        self.base_url = base_url
+        parsed_url = urlparse(base_url)
+        host = f"{parsed_url.scheme}://{parsed_url.netloc}/"
+        super().__init__(
+            host=host,
+            session=session,
+            read_timeout=REQUEST_TIMEOUT,
+            max_retries=max_retries,
+        )
+
+    def get_aws_costs_report(self, app: str) -> AwsReportCostResponse:
+        params = {
+            "cost_type": "calculated_amortized_cost",
+            "delta": "cost",
+            "filter[resolution]": "monthly",
+            "filter[tag:app]": app,
+            "filter[time_scope_units]": "month",
+            "filter[time_scope_value]": "-2",
+            "group_by[service]": "*",
+        }
+        response = self.session.request(
+            method="GET",
+            url=f"{self.base_url}/reports/aws/costs/",
+            params=params,
+            timeout=self.read_timeout,
+        )
+        response.raise_for_status()
+        return AwsReportCostResponse.parse_obj(response.json())
+
+    def get_openshift_costs_report(
+        self,
+        cluster: str,
+        project: str,
+    ) -> OpenShiftReportCostResponse:
+        params = {
+            "delta": "cost",
+            "filter[resolution]": "monthly",
+            "filter[cluster]": cluster,
+            "filter[exact:project]": project,
+            "filter[time_scope_units]": "month",
+            "filter[time_scope_value]": "-2",
+            "group_by[project]": "*",
+        }
+        response = self.session.request(
+            method="GET",
+            url=f"{self.base_url}/reports/openshift/costs/",
+            params=params,
+            timeout=self.read_timeout,
+        )
+        response.raise_for_status()
+        return OpenShiftReportCostResponse.parse_obj(response.json())
+
+    def get_openshift_cost_optimization_report(
+        self,
+        cluster: str,
+        project: str,
+    ) -> OpenShiftCostOptimizationReportResponse:
+        params = {
+            "cluster": cluster,
+            "project": project,
+            "limit": str(PAGE_LIMIT),
+            "memory-unit": MEMORY_UNIT,
+            "cpu-unit": CPU_UNIT,
+        }
+        response = self.session.request(
+            method="GET",
+            url=f"{self.base_url}/recommendations/openshift",
+            params=params,
+            timeout=self.read_timeout,
+        )
+        response.raise_for_status()
+
+        data = self._get_paginated(response)
+        return OpenShiftCostOptimizationReportResponse.parse_obj(data)
+
+    def _get_paginated(
+        self,
+        response: Response,
+    ) -> dict[str, list]:
+        body = response.json()
+        data = body.get("data", [])
+
+        while next_url := body.get("links", {}).get("next"):
+            r = self.session.request(
+                method="GET",
+                url=urljoin(self.host, next_url),
+                timeout=self.read_timeout,
+            )
+            r.raise_for_status()
+            body = r.json()
+            data.extend(body.get("data", []))
+
+        return {"data": data}
+
+    @classmethod
+    def create_from_secret(
+        cls,
+        secret: Mapping[str, str],
+    ) -> Self:
+        return cls(
+            base_url=secret["api_base_url"],
+            token_url=secret["token_url"],
+            client_id=secret["client_id"],
+            client_secret=secret["client_secret"],
+            scope=secret["scope"].split(" "),
+        )

tools/cli_commands/cost_report/model.py

@@ -0,0 +1,49 @@
+from decimal import Decimal
+
+from pydantic import BaseModel
+
+
+class ReportItem(BaseModel):
+    name: str
+    delta_value: Decimal
+    delta_percent: float | None
+    total: Decimal
+
+
+class ChildAppReport(BaseModel):
+    name: str
+    total: Decimal
+
+
+class Report(BaseModel):
+    app_name: str
+    child_apps: list[ChildAppReport]
+    child_apps_total: Decimal
+    date: str
+    parent_app_name: str | None
+    items: list[ReportItem]
+    items_delta_percent: float | None
+    items_delta_value: Decimal
+    items_total: Decimal
+    total: Decimal
+
+
+class OptimizationReportItem(BaseModel):
+    cluster: str
+    project: str
+    workload: str
+    workload_type: str
+    container: str
+    current_cpu_limit: str | None
+    current_cpu_request: str | None
+    current_memory_limit: str | None
+    current_memory_request: str | None
+    recommend_cpu_limit: str | None
+    recommend_cpu_request: str | None
+    recommend_memory_limit: str | None
+    recommend_memory_request: str | None
+
+
+class OptimizationReport(BaseModel):
+    app_name: str
+    items: list[OptimizationReportItem]

tools/cli_commands/cost_report/openshift.py

@@ -0,0 +1,166 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from decimal import Decimal
+from typing import Self
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.cost_report.app_names import App, get_app_names
+from reconcile.typed_queries.cost_report.cost_namespaces import (
+    CostNamespace,
+    get_cost_namespaces,
+)
+from reconcile.utils import gql
+from tools.cli_commands.cost_report.cost_management_api import CostManagementApi
+from tools.cli_commands.cost_report.model import ChildAppReport, Report, ReportItem
+from tools.cli_commands.cost_report.response import OpenShiftReportCostResponse
+from tools.cli_commands.cost_report.util import (
+    fetch_cost_report_secret,
+    process_reports,
+)
+from tools.cli_commands.cost_report.view import render_openshift_cost_report
+
+THREAD_POOL_SIZE = 10
+
+
+class OpenShiftCostReportCommand:
+    def __init__(
+        self,
+        gql_api: gql.GqlApi,
+        cost_management_api: CostManagementApi,
+        thread_pool_size: int = THREAD_POOL_SIZE,
+    ) -> None:
+        self.gql_api = gql_api
+        self.cost_management_api = cost_management_api
+        self.thread_pool_size = thread_pool_size
+
+    def execute(self) -> str:
+        apps = self.get_apps()
+        cost_namespaces = self.get_cost_namespaces()
+        responses = self.get_reports(cost_namespaces)
+        reports = self.process_reports(apps, responses)
+        return self.render(reports)
+
+    def get_apps(self) -> list[App]:
+        return get_app_names(self.gql_api)
+
+    def get_cost_namespaces(self) -> list[CostNamespace]:
+        return get_cost_namespaces(self.gql_api)
+
+    def _get_report(
+        self,
+        cost_namespace: CostNamespace,
+    ) -> tuple[CostNamespace, OpenShiftReportCostResponse]:
+        cluster = (
+            cost_namespace.cluster_external_id
+            if cost_namespace.cluster_external_id is not None
+            else cost_namespace.cluster_name
+        )
+        response = self.cost_management_api.get_openshift_costs_report(
+            project=cost_namespace.name,
+            cluster=cluster,
+        )
+        return cost_namespace, response
+
+    def get_reports(
+        self,
+        cost_namespaces: Iterable[CostNamespace],
+    ) -> dict[CostNamespace, OpenShiftReportCostResponse]:
+        """
+        Fetch reports from cost management API
+        """
+        results = threaded.run(self._get_report, cost_namespaces, self.thread_pool_size)
+        return dict(results)
+
+    def process_reports(
+        self,
+        apps: Iterable[App],
+        responses: Mapping[CostNamespace, OpenShiftReportCostResponse],
+    ) -> dict[str, Report]:
+        """
+        Build reports with parent-child app tree.
+        """
+        app_responses = defaultdict(list)
+        for cost_namespace, response in responses.items():
+            app_responses[cost_namespace.app_name].append(response)
+        return process_reports(
+            apps,
+            app_responses,
+            report_builder=self._build_report,
+        )
+
+    @staticmethod
+    def render(
+        reports: Mapping[str, Report],
+    ) -> str:
+        return render_openshift_cost_report(reports=reports)
+
+    @staticmethod
+    def _build_report(
+        app_name: str,
+        parent_app_name: str,
+        child_apps: list[str],
+        reports: Mapping[str, Report],
+        response: list[OpenShiftReportCostResponse],
+    ) -> Report:
+        child_app_reports = [
+            ChildAppReport(
+                name=child_app,
+                total=reports[child_app].total,
+            )
+            for child_app in child_apps
+        ]
+        child_apps_total = Decimal(
+            sum(child_app.total for child_app in child_app_reports)
+        )
+
+        items = [
+            ReportItem(
+                name=f"{value.clusters[0]}/{project.project}",
+                delta_value=value.delta_value,
+                delta_percent=value.delta_percent,
+                total=value.cost.total.value,
+            )
+            for r in response
+            for data in r.data
+            for project in data.projects
+            if len(project.values) == 1 and (value := project.values[0]) is not None
+        ]
+
+        items_total = Decimal(sum(item.total for item in items))
+        items_delta_value = Decimal(sum(item.delta_value for item in items))
+        previous_items_total = items_total - items_delta_value
+        items_delta_percent = (
+            items_delta_value / previous_items_total * 100
+            if previous_items_total != 0
+            else None
+        )
+        total = items_total + child_apps_total
+        date = next(
+            (d for r in response for data in r.data if (d := data.date)),
+            "",
+        )
+
+        return Report(
+            app_name=app_name,
+            child_apps=child_app_reports,
+            child_apps_total=child_apps_total,
+            date=date,
+            parent_app_name=parent_app_name,
+            items_delta_value=items_delta_value,
+            items_delta_percent=items_delta_percent,
+            items_total=items_total,
+            total=total,
+            items=items,
+        )
+
+    @classmethod
+    def create(cls) -> Self:
+        gql_api = gql.get_api()
+        secret = fetch_cost_report_secret(gql_api)
+        cost_management_api = CostManagementApi.create_from_secret(secret)
+        return cls(
+            gql_api=gql_api,
+            cost_management_api=cost_management_api,
+            thread_pool_size=THREAD_POOL_SIZE,
+        )

tools/cli_commands/cost_report/openshift_cost_optimization.py

@@ -0,0 +1,187 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from typing import Self
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.cost_report.app_names import App, get_app_names
+from reconcile.typed_queries.cost_report.cost_namespaces import (
+    CostNamespace,
+    get_cost_namespaces,
+)
+from reconcile.utils import gql
+from tools.cli_commands.cost_report.cost_management_api import CostManagementApi
+from tools.cli_commands.cost_report.model import (
+    OptimizationReport,
+    OptimizationReportItem,
+)
+from tools.cli_commands.cost_report.response import (
+    OpenShiftCostOptimizationReportResponse,
+    OpenShiftCostOptimizationResponse,
+    ResourceConfigResponse,
+)
+from tools.cli_commands.cost_report.util import fetch_cost_report_secret
+from tools.cli_commands.cost_report.view import (
+    render_openshift_cost_optimization_report,
+)
+
+THREAD_POOL_SIZE = 10
+
+
+class OpenShiftCostOptimizationReportCommand:
+    def __init__(
+        self,
+        gql_api: gql.GqlApi,
+        cost_management_api: CostManagementApi,
+        thread_pool_size: int = THREAD_POOL_SIZE,
+    ) -> None:
+        self.gql_api = gql_api
+        self.cost_management_api = cost_management_api
+        self.thread_pool_size = thread_pool_size
+
+    def execute(self) -> str:
+        apps = self.get_apps()
+        cost_namespaces = self.get_cost_namespaces()
+        responses = self.get_reports(cost_namespaces)
+        reports = self.process_reports(apps, responses)
+        return self.render(reports)
+
+    def get_apps(self) -> list[App]:
+        return get_app_names(self.gql_api)
+
+    def get_cost_namespaces(self) -> list[CostNamespace]:
+        cost_namespaces = get_cost_namespaces(self.gql_api)
+        return [
+            n
+            for n in cost_namespaces
+            if n.labels.insights_cost_management_optimizations == "true"
+        ]
+
+    def get_reports(
+        self,
+        cost_namespaces: Iterable[CostNamespace],
+    ) -> dict[CostNamespace, OpenShiftCostOptimizationReportResponse]:
+        results = threaded.run(self._get_report, cost_namespaces, self.thread_pool_size)
+        return dict(results)
+
+    def process_reports(
+        self,
+        apps: Iterable[App],
+        responses: Mapping[CostNamespace, OpenShiftCostOptimizationReportResponse],
+    ) -> list[OptimizationReport]:
+        app_responses = defaultdict(list)
+        for cost_namespace, response in responses.items():
+            app_responses[cost_namespace.app_name].append(response)
+        return [
+            self._build_report(
+                app.name,
+                app_responses.get(app.name, []),
+            )
+            for app in apps
+        ]
+
+    def _get_report(
+        self,
+        cost_namespace: CostNamespace,
+    ) -> tuple[CostNamespace, OpenShiftCostOptimizationReportResponse]:
+        cluster = (
+            cost_namespace.cluster_external_id
+            if cost_namespace.cluster_external_id is not None
+            else cost_namespace.cluster_name
+        )
+        response = self.cost_management_api.get_openshift_cost_optimization_report(
+            project=cost_namespace.name,
+            cluster=cluster,
+        )
+        response.data = [
+            data
+            for data in response.data
+            if self._match_cost_namespace(data, cost_namespace)
+        ]
+        return cost_namespace, response
+
+    @staticmethod
+    def _match_cost_namespace(
+        response: OpenShiftCostOptimizationResponse,
+        cost_namespace: CostNamespace,
+    ) -> bool:
+        """
+        Exactly match the cost namespace from the response data.
+        Cost Management API returns fuzzy match on fields.
+        Client side filter is needed.
+
+        :param response: OpenShiftCostOptimizationResponse
+        :param cost_namespace: CostNamespace
+        :return: exactly match or not
+        """
+        if cluster_uuid := cost_namespace.cluster_external_id:
+            if response.cluster_uuid != cluster_uuid:
+                return False
+        elif response.cluster_alias != cost_namespace.cluster_name:
+            return False
+        return response.project == cost_namespace.name
+
+    @staticmethod
+    def render(
+        reports: Iterable[OptimizationReport],
+    ) -> str:
+        return render_openshift_cost_optimization_report(reports)
+
+    def _build_report(
+        self,
+        app_name: str,
+        responses: list[OpenShiftCostOptimizationReportResponse],
+    ) -> OptimizationReport:
+        return OptimizationReport(
+            app_name=app_name,
+            items=[self._build_report_item(data) for r in responses for data in r.data],
+        )
+
+    def _build_report_item(
+        self,
+        data: OpenShiftCostOptimizationResponse,
+    ) -> OptimizationReportItem:
+        current = data.recommendations.current
+        terms = data.recommendations.recommendation_terms
+        recommend = next(
+            engine.cost.config
+            for t in [terms.long_term, terms.medium_term, terms.short_term]
+            if (engine := t.recommendation_engines) is not None
+        )
+
+        return OptimizationReportItem(
+            cluster=data.cluster_alias,
+            project=data.project,
+            workload=data.workload,
+            workload_type=data.workload_type,
+            container=data.container,
+            current_cpu_limit=self._build_resource_config(current.limits.cpu),
+            current_cpu_request=self._build_resource_config(current.requests.cpu),
+            current_memory_limit=self._build_resource_config(current.limits.memory),
+            current_memory_request=self._build_resource_config(current.requests.memory),
+            recommend_cpu_limit=self._build_resource_config(recommend.limits.cpu),
+            recommend_cpu_request=self._build_resource_config(recommend.requests.cpu),
+            recommend_memory_limit=self._build_resource_config(recommend.limits.memory),
+            recommend_memory_request=self._build_resource_config(
+                recommend.requests.memory
+            ),
+        )
+
+    @staticmethod
+    def _build_resource_config(response: ResourceConfigResponse) -> str | None:
+        if response.amount is None:
+            return None
+        if response.format is None:
+            return str(round(response.amount))
+        return f"{round(response.amount)}{response.format}"
+
+    @classmethod
+    def create(cls) -> Self:
+        gql_api = gql.get_api()
+        secret = fetch_cost_report_secret(gql_api)
+        cost_management_api = CostManagementApi.create_from_secret(secret)
+        return cls(
+            gql_api=gql_api,
+            cost_management_api=cost_management_api,
+            thread_pool_size=THREAD_POOL_SIZE,
+        )

tools/cli_commands/cost_report/response.py

@@ -0,0 +1,124 @@
+from decimal import Decimal
+
+from pydantic import BaseModel
+
+
+class DeltaResponse(BaseModel):
+    value: Decimal
+    percent: float | None
+
+
+class MoneyResponse(BaseModel):
+    value: Decimal
+    units: str
+
+
+class CostTotalResponse(BaseModel):
+    total: MoneyResponse
+
+
+class TotalMetaResponse(BaseModel):
+    cost: CostTotalResponse
+
+
+class ReportMetaResponse(BaseModel):
+    delta: DeltaResponse
+    total: TotalMetaResponse
+
+
+class ServiceCostValueResponse(BaseModel):
+    delta_value: Decimal
+    delta_percent: float | None
+    cost: CostTotalResponse
+
+
+class ServiceCostResponse(BaseModel):
+    service: str
+    values: list[ServiceCostValueResponse]
+
+
+class CostResponse(BaseModel):
+    date: str
+    services: list[ServiceCostResponse]
+
+
+class AwsReportCostResponse(BaseModel):
+    meta: ReportMetaResponse
+    data: list[CostResponse]
+
+
+class ProjectCostValueResponse(BaseModel):
+    delta_value: Decimal
+    delta_percent: float | None
+    cost: CostTotalResponse
+    clusters: list[str]
+
+
+class ProjectCostResponse(BaseModel):
+    project: str
+    values: list[ProjectCostValueResponse]
+
+
+class OpenShiftCostResponse(BaseModel):
+    date: str
+    projects: list[ProjectCostResponse]
+
+
+class OpenShiftReportCostResponse(BaseModel):
+    meta: ReportMetaResponse
+    data: list[OpenShiftCostResponse]
+
+
+class ResourceConfigResponse(BaseModel):
+    amount: float | None = None
+    format: str | None = None
+
+
+class ResourceResponse(BaseModel):
+    cpu: ResourceConfigResponse
+    memory: ResourceConfigResponse
+
+
+class RecommendationResourcesResponse(BaseModel):
+    limits: ResourceResponse
+    requests: ResourceResponse
+
+
+class RecommendationEngineResponse(BaseModel):
+    config: RecommendationResourcesResponse
+    variation: RecommendationResourcesResponse
+
+
+class RecommendationEnginesResponse(BaseModel):
+    cost: RecommendationEngineResponse
+    performance: RecommendationEngineResponse
+
+
+class RecommendationTermResponse(BaseModel):
+    recommendation_engines: RecommendationEnginesResponse | None = None
+
+
+class RecommendationTermsResponse(BaseModel):
+    long_term: RecommendationTermResponse
+    medium_term: RecommendationTermResponse
+    short_term: RecommendationTermResponse
+
+
+class RecommendationsResponse(BaseModel):
+    current: RecommendationResourcesResponse
+    recommendation_terms: RecommendationTermsResponse
+
+
+class OpenShiftCostOptimizationResponse(BaseModel):
+    cluster_alias: str
+    cluster_uuid: str
+    container: str
+    id: str
+    project: str
+    recommendations: RecommendationsResponse
+    workload: str
+    workload_type: str
+
+
+class OpenShiftCostOptimizationReportResponse(BaseModel):
+    data: list[OpenShiftCostOptimizationResponse]