qontract-reconcile 0.10.1rc801__py3-none-any.whl → 0.10.1rc803__py3-none-any.whl

tools/cli_commands/cost_report/cost_management_api.py

@@ -1,12 +1,13 @@
-from typing import List
-
 from urllib3.util import Retry
 
 from reconcile.utils.oauth2_backend_application_session import (
     OAuth2BackendApplicationSession,
 )
 from reconcile.utils.rest_api_base import ApiBase
-from tools.cli_commands.cost_report.response import ReportCostResponse
+from tools.cli_commands.cost_report.response import (
+    OpenShiftReportCostResponse,
+    ReportCostResponse,
+)
 
 REQUEST_TIMEOUT = 60
 
@@ -18,7 +19,7 @@ class CostManagementApi(ApiBase):
         token_url: str,
         client_id: str,
         client_secret: str,
-        scope: List[str] | None = None,
+        scope: list[str] | None = None,
     ) -> None:
         session = OAuth2BackendApplicationSession(
             client_id=client_id,
@@ -56,3 +57,26 @@ class CostManagementApi(ApiBase):
         )
         response.raise_for_status()
         return ReportCostResponse.parse_obj(response.json())
+
+    def get_openshift_costs_report(
+        self,
+        cluster: str,
+        project: str,
+    ) -> OpenShiftReportCostResponse:
+        params = {
+            "delta": "cost",
+            "filter[resolution]": "monthly",
+            "filter[cluster]": cluster,
+            "filter[exact:project]": project,
+            "filter[time_scope_units]": "month",
+            "filter[time_scope_value]": "-2",
+            "group_by[project]": "*",
+        }
+        response = self.session.request(
+            method="GET",
+            url=f"{self.host}/reports/openshift/costs/",
+            params=params,
+            timeout=self.read_timeout,
+        )
+        response.raise_for_status()
+        return OpenShiftReportCostResponse.parse_obj(response.json())

tools/cli_commands/cost_report/model.py

@@ -1,11 +1,10 @@
 from decimal import Decimal
-from typing import List
 
 from pydantic import BaseModel
 
 
-class ServiceReport(BaseModel):
-    service: str
+class ReportItem(BaseModel):
+    name: str
     delta_value: Decimal
     delta_percent: float | None
     total: Decimal
@@ -18,12 +17,12 @@ class ChildAppReport(BaseModel):
 
 class Report(BaseModel):
     app_name: str
-    child_apps: List[ChildAppReport]
+    child_apps: list[ChildAppReport]
     child_apps_total: Decimal
     date: str
     parent_app_name: str | None
-    services: List[ServiceReport]
-    services_delta_percent: float | None
-    services_delta_value: Decimal
-    services_total: Decimal
+    items: list[ReportItem]
+    items_delta_percent: float | None
+    items_delta_value: Decimal
+    items_total: Decimal
     total: Decimal

tools/cli_commands/cost_report/openshift.py

@@ -0,0 +1,179 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping
+from decimal import Decimal
+from typing import Self
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.cost_report.app_names import App, get_app_names
+from reconcile.typed_queries.cost_report.cost_namespaces import (
+    CostNamespace,
+    get_cost_namespaces,
+)
+from reconcile.typed_queries.cost_report.settings import get_cost_report_settings
+from reconcile.utils import gql
+from reconcile.utils.secret_reader import create_secret_reader
+from tools.cli_commands.cost_report.cost_management_api import CostManagementApi
+from tools.cli_commands.cost_report.model import ChildAppReport, Report, ReportItem
+from tools.cli_commands.cost_report.response import OpenShiftReportCostResponse
+from tools.cli_commands.cost_report.util import process_reports
+from tools.cli_commands.cost_report.view import render_openshift_cost_report
+
+THREAD_POOL_SIZE = 10
+
+
+class OpenShiftCostReportCommand:
+    def __init__(
+        self,
+        gql_api: gql.GqlApi,
+        cost_management_api: CostManagementApi,
+        thread_pool_size: int = THREAD_POOL_SIZE,
+    ) -> None:
+        self.gql_api = gql_api
+        self.cost_management_api = cost_management_api
+        self.thread_pool_size = thread_pool_size
+
+    def execute(self) -> str:
+        apps = self.get_apps()
+        cost_namespaces = self.get_cost_namespaces()
+        responses = self.get_reports(cost_namespaces)
+        reports = self.process_reports(apps, responses)
+        return self.render(reports)
+
+    def get_apps(self) -> list[App]:
+        return get_app_names(self.gql_api)
+
+    def get_cost_namespaces(self) -> list[CostNamespace]:
+        return get_cost_namespaces(self.gql_api)
+
+    def _get_report(
+        self,
+        cost_namespace: CostNamespace,
+    ) -> tuple[CostNamespace, OpenShiftReportCostResponse]:
+        cluster = (
+            cost_namespace.cluster_external_id
+            if cost_namespace.cluster_external_id is not None
+            else cost_namespace.cluster_name
+        )
+        response = self.cost_management_api.get_openshift_costs_report(
+            project=cost_namespace.name,
+            cluster=cluster,
+        )
+        return cost_namespace, response
+
+    def get_reports(
+        self,
+        cost_namespaces: Iterable[CostNamespace],
+    ) -> dict[CostNamespace, OpenShiftReportCostResponse]:
+        """
+        Fetch reports from cost management API
+        """
+        results = threaded.run(self._get_report, cost_namespaces, self.thread_pool_size)
+        return dict(results)
+
+    def process_reports(
+        self,
+        apps: Iterable[App],
+        responses: Mapping[CostNamespace, OpenShiftReportCostResponse],
+    ) -> dict[str, Report]:
+        """
+        Build reports with parent-child app tree.
+        """
+        app_responses = defaultdict(list)
+        for cost_namespace, response in responses.items():
+            app_responses[cost_namespace.app_name].append(response)
+        return process_reports(
+            apps,
+            app_responses,
+            report_builder=self._build_report,
+        )
+
+    @staticmethod
+    def render(
+        reports: Mapping[str, Report],
+    ) -> str:
+        return render_openshift_cost_report(reports=reports)
+
+    @staticmethod
+    def _build_report(
+        app_name: str,
+        parent_app_name: str,
+        child_apps: list[str],
+        reports: Mapping[str, Report],
+        response: list[OpenShiftReportCostResponse],
+    ) -> Report:
+        child_app_reports = [
+            ChildAppReport(
+                name=child_app,
+                total=reports[child_app].total,
+            )
+            for child_app in child_apps
+        ]
+        child_apps_total = Decimal(
+            sum(child_app.total for child_app in child_app_reports)
+        )
+
+        items = [
+            ReportItem(
+                name=f"{value.clusters[0]}/{project.project}",
+                delta_value=value.delta_value,
+                delta_percent=value.delta_percent,
+                total=value.cost.total.value,
+            )
+            for r in response
+            for data in r.data
+            for project in data.projects
+            if len(project.values) == 1 and (value := project.values[0]) is not None
+        ]
+
+        items_total = Decimal(sum(item.total for item in items))
+        items_delta_value = Decimal(sum(item.delta_value for item in items))
+        previous_items_total = items_total - items_delta_value
+        items_delta_percent = (
+            items_delta_value / previous_items_total * 100
+            if previous_items_total != 0
+            else None
+        )
+        total = items_total + child_apps_total
+        date = next(
+            (d for r in response for data in r.data if (d := data.date)),
+            "",
+        )
+
+        return Report(
+            app_name=app_name,
+            child_apps=child_app_reports,
+            child_apps_total=child_apps_total,
+            date=date,
+            parent_app_name=parent_app_name,
+            items_delta_value=items_delta_value,
+            items_delta_percent=items_delta_percent,
+            items_total=items_total,
+            total=total,
+            items=items,
+        )
+
+    @classmethod
+    def create(
+        cls,
+    ) -> Self:
+        gql_api = gql.get_api()
+        vault_settings = get_app_interface_vault_settings(gql_api.query)
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+        cost_report_settings = get_cost_report_settings(gql_api)
+        secret = secret_reader.read_all_secret(cost_report_settings.credentials)
+        cost_management_api = CostManagementApi(
+            base_url=secret["api_base_url"],
+            token_url=secret["token_url"],
+            client_id=secret["client_id"],
+            client_secret=secret["client_secret"],
+            scope=secret["scope"].split(" "),
+        )
+        return cls(
+            gql_api=gql_api,
+            cost_management_api=cost_management_api,
+            thread_pool_size=THREAD_POOL_SIZE,
+        )

tools/cli_commands/cost_report/response.py

@@ -1,5 +1,4 @@
 from decimal import Decimal
-from typing import List
 
 from pydantic import BaseModel
 
@@ -35,14 +34,36 @@ class ServiceCostValueResponse(BaseModel):
 
 class ServiceCostResponse(BaseModel):
     service: str
-    values: List[ServiceCostValueResponse]
+    values: list[ServiceCostValueResponse]
 
 
 class CostResponse(BaseModel):
     date: str
-    services: List[ServiceCostResponse]
+    services: list[ServiceCostResponse]
 
 
 class ReportCostResponse(BaseModel):
     meta: ReportMetaResponse
-    data: List[CostResponse]
+    data: list[CostResponse]
+
+
+class ProjectCostValueResponse(BaseModel):
+    delta_value: Decimal
+    delta_percent: float | None
+    cost: CostTotalResponse
+    clusters: list[str]
+
+
+class ProjectCostResponse(BaseModel):
+    project: str
+    values: list[ProjectCostValueResponse]
+
+
+class OpenShiftCostResponse(BaseModel):
+    date: str
+    projects: list[ProjectCostResponse]
+
+
+class OpenShiftReportCostResponse(BaseModel):
+    meta: ReportMetaResponse
+    data: list[OpenShiftCostResponse]

tools/cli_commands/cost_report/util.py

@@ -0,0 +1,59 @@
+from collections import defaultdict
+from collections.abc import Callable, Iterable, Mapping, MutableMapping
+from typing import Any
+
+from reconcile.typed_queries.cost_report.app_names import App
+from tools.cli_commands.cost_report.model import Report
+
+
+def dfs_build_reports(
+    app_name: str,
+    parent_app_name: str | None,
+    child_apps_by_parent: Mapping[str | None, list[str]],
+    responses: Mapping[str, Any],
+    reports: MutableMapping[str, Report],
+    report_builder: Callable[..., Report],
+) -> None:
+    """
+    Depth-first search to build the reports. Build leaf nodes first to ensure total is calculated correctly.
+    """
+    child_apps = child_apps_by_parent.get(app_name, [])
+    for child_app in child_apps:
+        dfs_build_reports(
+            app_name=child_app,
+            parent_app_name=app_name,
+            child_apps_by_parent=child_apps_by_parent,
+            responses=responses,
+            reports=reports,
+            report_builder=report_builder,
+        )
+    reports[app_name] = report_builder(
+        app_name=app_name,
+        parent_app_name=parent_app_name,
+        child_apps=child_apps,
+        reports=reports,
+        response=responses[app_name],
+    )
+
+
+def process_reports(
+    apps: Iterable[App],
+    responses: Mapping[str, Any],
+    report_builder: Callable[..., Report],
+) -> dict[str, Report]:
+    child_apps_by_parent = defaultdict(list)
+    for app in apps:
+        child_apps_by_parent[app.parent_app_name].append(app.name)
+
+    reports: dict[str, Report] = {}
+    root_apps = child_apps_by_parent.get(None, [])
+    for app_name in root_apps:
+        dfs_build_reports(
+            app_name,
+            None,
+            child_apps_by_parent=child_apps_by_parent,
+            responses=responses,
+            reports=reports,
+            report_builder=report_builder,
+        )
+    return reports

tools/cli_commands/cost_report/view.py

@@ -1,4 +1,4 @@
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from decimal import Decimal
 from typing import Any
 
@@ -15,11 +15,15 @@ LAYOUT = """\
 {cost_breakdown}\
 """
 
-HEADER = """\
+AWS_HEADER = """\
 # AWS Cost Report
 """
 
-SUMMARY = """\
+OPENSHIFT_HEADER = """\
+# OpenShift Cost Report
+"""
+
+AWS_SUMMARY = """\
 ## Summary
 
 Total AWS Cost for {date}: {total_cost}
@@ -29,6 +33,16 @@ Total AWS Cost for {date}: {total_cost}
 ```
 """
 
+OPENSHIFT_SUMMARY = """\
+## Summary
+
+Total OpenShift Cost for {date}: {total_cost}
+
+```json:table
+{json_table}
+```
+"""
+
 MONTH_OVER_MONTH_CHANGE = """\
 ## Month Over Month Change
 
@@ -52,7 +66,7 @@ APP = """\
 """
 
 AWS_SERVICES_COST = """\
-AWS Services Cost: {services_total}, {services_delta_value}{services_delta_percent} \
+AWS Services Cost: {items_total}, {items_delta_value}{items_delta_percent} \
 compared to previous month.
 View in [Cost Management Console]({cost_management_console_url}).
 
@@ -61,6 +75,15 @@ View in [Cost Management Console]({cost_management_console_url}).
 ```
 """
 
+OPENSHIFT_WORKLOADS_COST = """\
+OpenShift Workloads Cost: {items_total}, {items_delta_value}{items_delta_percent} \
+compared to previous month.
+
+```json:table
+{json_table}
+```
+"""
+
 COST_MANAGEMENT_CONSOLE_EXPLORE_URL = (
     "{base_url}/explorer?"
     "dateRangeType=previous_month&"
@@ -100,7 +123,7 @@ class JsonTable(BaseModel):
 class SummaryItem(BaseModel):
     name: str
     child_apps_total: Decimal
-    services_total: Decimal
+    items_total: Decimal
     total: Decimal
 
 
@@ -111,6 +134,18 @@ class MonthOverMonthChangeItem(BaseModel):
     total: Decimal
 
 
+class ViewReportItem(BaseModel):
+    name: str
+    delta_value: Decimal
+    delta_percent: float | None
+    total: Decimal
+
+
+class ViewChildAppReport(BaseModel):
+    name: str
+    total: Decimal
+
+
 def format_cost_value(value: Decimal) -> str:
     return f"${value:,.2f}"
 
@@ -132,7 +167,10 @@ def get_date(reports: Mapping[str, Report]) -> str:
     return next((d for report in reports.values() if (d := report.date)), "")
 
 
-def render_summary(reports: Mapping[str, Report]) -> str:
+def render_summary(
+    template: str,
+    reports: Mapping[str, Report],
+) -> str:
     root_apps = {
         name: report
         for name, report in reports.items()
@@ -143,7 +181,7 @@ def render_summary(reports: Mapping[str, Report]) -> str:
         SummaryItem(
             name=name,
             child_apps_total=round(report.child_apps_total, 2),
-            services_total=round(report.services_total, 2),
+            items_total=round(report.items_total, 2),
             total=round(report.total, 2),
         )
         for name, report in root_apps.items()
@@ -153,12 +191,12 @@ def render_summary(reports: Mapping[str, Report]) -> str:
         items=sorted(summary_items, key=lambda item: item.total, reverse=True),
         fields=[
             TableField(key="name", label="Name", sortable=True),
-            TableField(key="services_total", label="Self App ($)", sortable=True),
+            TableField(key="items_total", label="Self App ($)", sortable=True),
             TableField(key="child_apps_total", label="Child Apps ($)", sortable=True),
             TableField(key="total", label="Total ($)", sortable=True),
         ],
     )
-    return SUMMARY.format(
+    return template.format(
         date=get_date(reports),
         total_cost=format_cost_value(total_cost),
         json_table=json_table.json(indent=2),
@@ -169,13 +207,13 @@ def render_month_over_month_change(reports: Mapping[str, Report]) -> str:
     items = [
         MonthOverMonthChangeItem(
             name=name,
-            delta_value=round(report.services_delta_value, 2),
+            delta_value=round(report.items_delta_value, 2),
             delta_percent=(
-                round(report.services_delta_percent, 2)
-                if report.services_delta_percent is not None
+                round(report.items_delta_percent, 2)
+                if report.items_delta_percent is not None
                 else None
             ),
-            total=round(report.services_total, 2),
+            total=round(report.items_total, 2),
         )
         for name, report in reports.items()
     ]
@@ -212,46 +250,60 @@ def render_aws_services_cost(
     report: Report,
     cost_management_console_base_url: str,
 ) -> str:
-    services = [
-        s.copy(
-            update={
-                "delta_value": round(s.delta_value, 2),
-                "delta_percent": round(s.delta_percent, 2)
-                if s.delta_percent is not None
-                else None,
-                "total": round(s.total, 2),
-            }
+    json_table = _build_items_cost_json_table(report, name_label="Service")
+    return AWS_SERVICES_COST.format(
+        cost_management_console_url=build_cost_management_console_url(
+            cost_management_console_base_url,
+            report.app_name,
+        ),
+        items_total=format_cost_value(report.items_total),
+        items_delta_value=format_delta_value(report.items_delta_value),
+        items_delta_percent=format_delta_percent(report.items_delta_percent),
+        json_table=json_table.json(indent=2),
+    )
+
+
+def render_openshift_workloads_cost(
+    report: Report,
+) -> str:
+    json_table = _build_items_cost_json_table(report, name_label="Cluster/Namespace")
+    return OPENSHIFT_WORKLOADS_COST.format(
+        items_total=format_cost_value(report.items_total),
+        items_delta_value=format_delta_value(report.items_delta_value),
+        items_delta_percent=format_delta_percent(report.items_delta_percent),
+        json_table=json_table.json(indent=2),
+    )
+
+
+def _build_items_cost_json_table(report: Report, name_label: str) -> JsonTable:
+    items = [
+        ViewReportItem(
+            name=s.name,
+            delta_value=round(s.delta_value, 2),
+            delta_percent=round(s.delta_percent, 2)
+            if s.delta_percent is not None
+            else None,
+            total=round(s.total, 2),
         )
-        for s in report.services
+        for s in report.items
     ]
-    json_table = JsonTable(
+    return JsonTable(
         filter=True,
-        items=sorted(services, key=lambda service: service.total, reverse=True),
+        items=sorted(items, key=lambda item: item.total, reverse=True),
         fields=[
-            TableField(key="service", label="Service", sortable=True),
+            TableField(key="name", label=name_label, sortable=True),
             TableField(key="delta_value", label="Change ($)", sortable=True),
             TableField(key="delta_percent", label="Change (%)", sortable=True),
             TableField(key="total", label="Total ($)", sortable=True),
         ],
     )
-    return AWS_SERVICES_COST.format(
-        cost_management_console_url=build_cost_management_console_url(
-            cost_management_console_base_url,
-            report.app_name,
-        ),
-        services_total=format_cost_value(report.services_total),
-        services_delta_value=format_delta_value(report.services_delta_value),
-        services_delta_percent=format_delta_percent(report.services_delta_percent),
-        json_table=json_table.json(indent=2),
-    )
 
 
 def render_child_apps_cost(report: Report) -> str:
     child_apps = [
-        app.copy(
-            update={
-                "total": round(app.total, 2),
-            }
+        ViewChildAppReport(
+            name=app.name,
+            total=round(app.total, 2),
         )
         for app in report.child_apps
     ]
@@ -278,16 +330,12 @@ def render_total_cost(report: Report) -> str:
 def render_app_cost(
     name: str,
     report: Report,
-    cost_management_console_base_url: str,
+    item_cost_renderer: Callable[..., str],
+    **kwargs: Any,
 ) -> str:
     cost_details_sections = []
-    if report.services:
-        cost_details_sections.append(
-            render_aws_services_cost(
-                report,
-                cost_management_console_base_url,
-            )
-        )
+    if report.items:
+        cost_details_sections.append(item_cost_renderer(report=report, **kwargs))
     if report.child_apps:
         cost_details_sections.append(render_child_apps_cost(report))
         cost_details_sections.append(render_total_cost(report))
@@ -302,13 +350,15 @@ def render_app_cost(
 
 def render_cost_breakdown(
     reports: Mapping[str, Report],
-    cost_management_console_base_url: str,
+    item_cost_renderer: Callable[..., str],
+    **kwargs: Any,
 ) -> str:
     apps = "\n".join(
         render_app_cost(
-            name,
-            report,
-            cost_management_console_base_url,
+            name=name,
+            report=report,
+            item_cost_renderer=item_cost_renderer,
+            **kwargs,
         )
         for name, report in sorted(
             reports.items(),
@@ -318,16 +368,31 @@ def render_cost_breakdown(
     return COST_BREAKDOWN.format(apps=apps)
 
 
-def render_report(
+def render_aws_cost_report(
     reports: Mapping[str, Report],
     cost_management_console_base_url: str,
 ) -> str:
     return LAYOUT.format(
-        header=HEADER,
-        summary=render_summary(reports),
+        header=AWS_HEADER,
+        summary=render_summary(AWS_SUMMARY, reports),
         month_over_month_change=render_month_over_month_change(reports),
         cost_breakdown=render_cost_breakdown(
             reports,
-            cost_management_console_base_url,
+            item_cost_renderer=render_aws_services_cost,
+            cost_management_console_base_url=cost_management_console_base_url,
+        ),
+    )
+
+
+def render_openshift_cost_report(
+    reports: Mapping[str, Report],
+) -> str:
+    return LAYOUT.format(
+        header=OPENSHIFT_HEADER,
+        summary=render_summary(OPENSHIFT_SUMMARY, reports),
+        month_over_month_change=render_month_over_month_change(reports),
+        cost_breakdown=render_cost_breakdown(
+            reports,
+            item_cost_renderer=render_openshift_workloads_cost,
         ),
     )