qontract-reconcile 0.10.1rc696__py3-none-any.whl → 0.10.1rc702__py3-none-any.whl

reconcile/utils/state.py

@@ -1,9 +1,10 @@
+import contextlib
 import json
 import logging
 import os
 from abc import abstractmethod
-from collections.abc import Callable, Mapping
-from types import TracebackType
+from collections.abc import Callable, Generator, Mapping
+from dataclasses import dataclass, field
 from typing import (
     Any,
     Optional,
@@ -212,6 +213,10 @@ def acquire_state_settings(
     )
 
 
+class AbortStateTransaction(Exception):
+    """Raise to abort a state transaction."""
+
+
 class State:
     """
     A state object to be used by stateful integrations.
@@ -405,52 +410,51 @@ class State:
     def __setitem__(self, key: str, value: Any) -> None:
         self._set(key, value)
 
-    def transaction(self, key: str, value: Any) -> "_TransactionContext":
+    @contextlib.contextmanager
+    def transaction(
+        self, key: str, value: Any = None
+    ) -> Generator["TransactionStateObj", None, None]:
         """Get a context manager to set the key in the state if no exception occurs.
 
+        You can set the value either via the value parameter or by setting the value attribute of the returned object.
+        If both are provided, the value attribute of the state object will take precedence.
+
         Attention!
 
         This is not a locking mechanism. It is a way to ensure that a key is set in the state if no exception occurs.
         This method is not thread-safe nor multi-process-safe! There is no locking mechanism in place.
         """
-        return _TransactionContext(self, key, value)
+        try:
+            _current_value = self[key]
+        except KeyError:
+            _current_value = None
+        state_obj = TransactionStateObj(key, value=_current_value)
+        try:
+            yield state_obj
+        except AbortStateTransaction:
+            return
+        else:
+            if state_obj.changed and state_obj.value != _current_value:
+                self[state_obj.key] = state_obj.value
+            elif value is not None and state_obj.value != value:
+                self[state_obj.key] = value
 
 
-class _TransactionContext:
-    """A context manager to set a key in the state if no exception occurs."""
+@dataclass
+class TransactionStateObj:
+    """Represents a transistion state object with a key and a value."""
 
-    def __init__(
-        self,
-        state: State,
-        key: str,
-        value: Any,
-    ):
-        self.state = state
-        self.key = key
-        self.value = value
+    key: str
+    value: Any = None
+    _init_value: Any = field(init=False)
 
-    def __enter__(self) -> bool:
-        """Return True if the key exists in the state, False otherwise.
+    def __post_init__(self) -> None:
+        self._init_value = self.value
 
-        Cache the previous value to avoid unnecessary updates.
-        """
-        self._previous_value = None
-        try:
-            self._previous_value = self.state[self.key]
-            return True
-        except KeyError:
-            return False
+    @property
+    def changed(self) -> bool:
+        return self.value != self._init_value
 
-    def __exit__(
-        self,
-        exc_type: type[BaseException] | None,
-        exc_value: BaseException | None,
-        traceback: TracebackType | None,
-    ) -> None:
-        if exc_type:
-            # if an exception occurred, we don't want to write to the state
-            return
-        if self._previous_value == self.value:
-            # if the value didn't change, we don't want to write to the state
-            return
-        self.state[self.key] = self.value
+    @property
+    def exists(self) -> bool:
+        return self._init_value is not None

tools/cli_commands/cost_report/command.py

@@ -0,0 +1,172 @@
+from collections import defaultdict
+from collections.abc import Iterable, Mapping, MutableMapping
+from decimal import Decimal
+from typing import List, Self, Tuple
+
+from sretoolbox.utils import threaded
+
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.cost_report.app_names import App, get_app_names
+from reconcile.typed_queries.cost_report.settings import get_cost_report_settings
+from reconcile.utils import gql
+from reconcile.utils.secret_reader import create_secret_reader
+from tools.cli_commands.cost_report.cost_management_api import CostManagementApi
+from tools.cli_commands.cost_report.model import ChildAppReport, Report, ServiceReport
+from tools.cli_commands.cost_report.response import ReportCostResponse
+from tools.cli_commands.cost_report.view import render_report
+
+THREAD_POOL_SIZE = 10
+
+
+class CostReportCommand:
+    def __init__(
+        self,
+        gql_api: gql.GqlApi,
+        cost_management_api: CostManagementApi,
+        cost_management_console_base_url: str,
+    ) -> None:
+        self.gql_api = gql_api
+        self.cost_management_api = cost_management_api
+        self.cost_management_console_base_url = cost_management_console_base_url
+
+    def execute(self) -> str:
+        apps = self.get_apps()
+        reports = self.get_reports(apps)
+        return self.render(reports)
+
+    def get_apps(self) -> list[App]:
+        """
+        Get all apps from the gql API.
+        """
+        return get_app_names(self.gql_api)
+
+    def _fetch_report(self, app: App) -> Tuple[str, ReportCostResponse]:
+        return app.name, self.cost_management_api.get_aws_costs_report(app.name)
+
+    def _fetch_reports(self, apps: Iterable[App]) -> dict[str, ReportCostResponse]:
+        results = threaded.run(self._fetch_report, apps, THREAD_POOL_SIZE)
+        return dict(results)
+
+    def get_reports(self, apps: Iterable[App]) -> dict[str, Report]:
+        """
+        Fetch reports from cost management API and build reports with parent-child app tree.
+        """
+        responses = self._fetch_reports(apps)
+
+        child_apps_by_parent = defaultdict(list)
+        for app in apps:
+            child_apps_by_parent[app.parent_app_name].append(app.name)
+
+        reports: dict[str, Report] = {}
+        root_apps = child_apps_by_parent.get(None, [])
+        for app_name in root_apps:
+            self._dfs_reports(
+                app_name,
+                None,
+                child_apps_by_parent=child_apps_by_parent,
+                responses=responses,
+                reports=reports,
+            )
+        return reports
+
+    def render(self, reports: Mapping[str, Report]) -> str:
+        return render_report(
+            reports=reports,
+            cost_management_console_base_url=self.cost_management_console_base_url,
+        )
+
+    def _dfs_reports(
+        self,
+        app_name: str,
+        parent_app_name: str | None,
+        child_apps_by_parent: Mapping[str | None, list[str]],
+        responses: Mapping[str, ReportCostResponse],
+        reports: MutableMapping[str, Report],
+    ) -> None:
+        """
+        Depth-first search to build the reports. Build leaf nodes first to ensure total is calculated correctly.
+        """
+        child_apps = child_apps_by_parent.get(app_name, [])
+        for child_app in child_apps:
+            self._dfs_reports(
+                app_name=child_app,
+                parent_app_name=app_name,
+                child_apps_by_parent=child_apps_by_parent,
+                responses=responses,
+                reports=reports,
+            )
+        reports[app_name] = self._build_report(
+            app_name=app_name,
+            parent_app_name=parent_app_name,
+            child_apps=child_apps,
+            reports=reports,
+            response=responses[app_name],
+        )
+
+    @staticmethod
+    def _build_report(
+        app_name: str,
+        parent_app_name: str | None,
+        child_apps: List[str],
+        reports: Mapping[str, Report],
+        response: ReportCostResponse,
+    ) -> Report:
+        child_app_reports = [
+            ChildAppReport(
+                name=child_app,
+                total=reports[child_app].total,
+            )
+            for child_app in child_apps
+        ]
+        child_apps_total = Decimal(
+            sum(child_app.total for child_app in child_app_reports)
+        )
+        services_total = response.meta.total.cost.total.value
+        total = services_total + child_apps_total
+        date = next((d for data in response.data if (d := data.date)), "")
+        return Report(
+            app_name=app_name,
+            child_apps=child_app_reports,
+            child_apps_total=child_apps_total,
+            date=date,
+            parent_app_name=parent_app_name,
+            services_delta_value=response.meta.delta.value,
+            services_delta_percent=response.meta.delta.percent,
+            services_total=services_total,
+            total=total,
+            services=[
+                ServiceReport(
+                    service=service.service,
+                    delta_value=value.delta_value,
+                    delta_percent=value.delta_percent,
+                    total=value.cost.total.value,
+                )
+                for data in response.data
+                for service in data.services
+                if len(service.values) == 1 and (value := service.values[0]) is not None
+            ],
+        )
+
+    @classmethod
+    def create(
+        cls,
+    ) -> Self:
+        gql_api = gql.get_api()
+        vault_settings = get_app_interface_vault_settings(gql_api.query)
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+        cost_report_settings = get_cost_report_settings(gql_api)
+        secret = secret_reader.read_all_secret(cost_report_settings.credentials)
+        cost_management_api = CostManagementApi(
+            base_url=secret["api_base_url"],
+            token_url=secret["token_url"],
+            client_id=secret["client_id"],
+            client_secret=secret["client_secret"],
+            scope=secret["scope"].split(" "),
+        )
+        return cls(
+            gql_api=gql_api,
+            cost_management_api=cost_management_api,
+            cost_management_console_base_url=secret["console_base_url"],
+        )

tools/cli_commands/cost_report/cost_management_api.py

@@ -0,0 +1,57 @@
+from typing import Any, List, Self
+
+from reconcile.utils.oauth2_backend_application_session import (
+    OAuth2BackendApplicationSession,
+)
+from tools.cli_commands.cost_report.response import ReportCostResponse
+
+REQUEST_TIMEOUT = 60
+
+
+class CostManagementApi:
+    def __init__(
+        self,
+        base_url: str,
+        token_url: str,
+        client_id: str,
+        client_secret: str,
+        scope: List[str] | None = None,
+        request_timeout: int | None = REQUEST_TIMEOUT,
+    ) -> None:
+        self.base_url = base_url
+        self.request_timeout = request_timeout
+        self.session = OAuth2BackendApplicationSession(
+            client_id=client_id,
+            client_secret=client_secret,
+            token_url=token_url,
+            scope=scope,
+        )
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        self.cleanup()
+
+    def cleanup(self) -> None:
+        self.session.close()
+
+    def get_aws_costs_report(self, app: str) -> ReportCostResponse:
+        params = {
+            "cost_type": "calculated_amortized_cost",
+            "delta": "cost",
+            "filter[resolution]": "monthly",
+            "filter[tag:app]": app,
+            "filter[time_scope_units]": "month",
+            "filter[time_scope_value]": "-2",
+            "group_by[service]": "*",
+        }
+        response = self.session.request(
+            method="GET",
+            url=f"{self.base_url}/reports/aws/costs/",
+            headers={"Content-Type": "application/json"},
+            params=params,
+            timeout=self.request_timeout,
+        )
+        response.raise_for_status()
+        return ReportCostResponse.parse_obj(response.json())

tools/cli_commands/cost_report/model.py

@@ -0,0 +1,29 @@
+from decimal import Decimal
+from typing import List
+
+from pydantic import BaseModel
+
+
+class ServiceReport(BaseModel):
+    service: str
+    delta_value: Decimal
+    delta_percent: float | None
+    total: Decimal
+
+
+class ChildAppReport(BaseModel):
+    name: str
+    total: Decimal
+
+
+class Report(BaseModel):
+    app_name: str
+    child_apps: List[ChildAppReport]
+    child_apps_total: Decimal
+    date: str
+    parent_app_name: str | None
+    services: List[ServiceReport]
+    services_delta_percent: float | None
+    services_delta_value: Decimal
+    services_total: Decimal
+    total: Decimal

tools/cli_commands/cost_report/response.py

@@ -0,0 +1,48 @@
+from decimal import Decimal
+from typing import List
+
+from pydantic import BaseModel
+
+
+class DeltaResponse(BaseModel):
+    value: Decimal
+    percent: float | None
+
+
+class MoneyResponse(BaseModel):
+    value: Decimal
+    units: str
+
+
+class CostTotalResponse(BaseModel):
+    total: MoneyResponse
+
+
+class TotalMetaResponse(BaseModel):
+    cost: CostTotalResponse
+
+
+class ReportMetaResponse(BaseModel):
+    delta: DeltaResponse
+    total: TotalMetaResponse
+
+
+class ServiceCostValueResponse(BaseModel):
+    delta_value: Decimal
+    delta_percent: float | None
+    cost: CostTotalResponse
+
+
+class ServiceCostResponse(BaseModel):
+    service: str
+    values: List[ServiceCostValueResponse]
+
+
+class CostResponse(BaseModel):
+    date: str
+    services: List[ServiceCostResponse]
+
+
+class ReportCostResponse(BaseModel):
+    meta: ReportMetaResponse
+    data: List[CostResponse]