qontract-reconcile 0.10.2.dev226__tar.gz → 0.10.2.dev228__tar.gz

{qontract_reconcile-0.10.2.dev226 → qontract_reconcile-0.10.2.dev228}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev226
+Version: 0.10.2.dev228
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev226 → qontract_reconcile-0.10.2.dev228}/reconcile/openshift_saas_deploy_trigger_base.py

@@ -38,7 +38,7 @@ from reconcile.utils.saasherder import (
     TriggerSpecUnion,
 )
 from reconcile.utils.saasherder.interfaces import SaasPipelinesProviderTekton
-from reconcile.utils.saasherder.models import TriggerTypes
+from reconcile.utils.saasherder.models import TriggerSpecConfig, TriggerTypes
 from reconcile.utils.secret_reader import create_secret_reader
 from reconcile.utils.sharding import is_in_shard
 from reconcile.utils.state import init_state
@@ -267,6 +267,10 @@ def _trigger_tekton(
         )
         return False
 
+    target_ref = None
+    if isinstance(spec, TriggerSpecConfig):
+        target_ref = spec.target_ref
+
     tkn_trigger_resource, tkn_name = _construct_tekton_trigger_resource(
         spec.saas_file_name,
         spec.env_name,
@@ -278,6 +282,7 @@ def _trigger_tekton(
         integration_version,
         saasherder.include_trigger_trace,
         spec.reason,
+        target_ref,
     )
 
     error = False
@@ -334,6 +339,7 @@ def _construct_tekton_trigger_resource(
     integration_version: str,
     include_trigger_trace: bool,
     reason: str | None,
+    target_ref: str | None,
 ) -> tuple[OR, str]:
     """Construct a resource (PipelineRun) to trigger a deployment via Tekton.
 
@@ -348,6 +354,7 @@ def _construct_tekton_trigger_resource(
         integration_version (string): Version of calling integration
         include_trigger_trace (bool): Should include traces of the triggering integration and reason
         reason (string): The reason this trigger was created
+        target_ref (string): the SHA ref of the target
 
     Returns:
         OpenshiftResource: OpenShift resource to be applied
@@ -385,6 +392,7 @@ def _construct_tekton_trigger_resource(
             "labels": {
                 "qontract.saas_file_name": saas_file_name,
                 "qontract.env_name": env_name,
+                "qontract.target_ref": target_ref or "",
             },
         },
         "spec": {

qontract_reconcile-0.10.2.dev228/reconcile/status_board.py

@@ -0,0 +1,473 @@
+import logging
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from collections.abc import Iterable, Mapping
+from enum import Enum
+from typing import (
+    Optional,
+)
+
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.slo_documents.slo_documents import SLODocumentV1
+from reconcile.gql_definitions.status_board.status_board import StatusBoardV1
+from reconcile.typed_queries.slo_documents import get_slo_documents
+from reconcile.typed_queries.status_board import (
+    get_selected_app_names,
+    get_status_board,
+)
+from reconcile.utils.differ import diff_mappings
+from reconcile.utils.ocm.status_board import (
+    ApplicationOCMSpec,
+    BaseOCMSpec,
+    ServiceMetadataSpec,
+    ServiceOCMSpec,
+    create_application,
+    create_product,
+    create_service,
+    delete_application,
+    delete_product,
+    delete_service,
+    get_application_services,
+    get_managed_products,
+    get_product_applications,
+    update_service,
+)
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+    init_ocm_base_client,
+)
+from reconcile.utils.runtime.integration import QontractReconcileIntegration
+
+QONTRACT_INTEGRATION = "status-board-exporter"
+
+
+class Action(Enum):
+    create = "create"
+    update = "update"
+    delete = "delete"
+
+
+class AbstractStatusBoard(ABC, BaseModel):
+    """Abstract class for upgrade policies
+    Used to create and delete upgrade policies in OCM."""
+
+    id: str | None
+    name: str
+    fullname: str
+
+    @abstractmethod
+    def create(self, ocm: OCMBaseClient) -> None:
+        pass
+
+    @abstractmethod
+    def update(self, ocm: OCMBaseClient) -> None:
+        pass
+
+    @abstractmethod
+    def delete(self, ocm: OCMBaseClient) -> None:
+        pass
+
+    @abstractmethod
+    def summarize(self) -> str:
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def get_priority() -> int:
+        pass
+
+    @abstractmethod
+    def to_ocm_spec(self) -> BaseOCMSpec:
+        pass
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, AbstractStatusBoard):
+            return NotImplemented
+        return self.name == other.name and self.fullname == other.fullname
+
+
+class Product(AbstractStatusBoard):
+    applications: list["Application"] | None
+
+    def create(self, ocm: OCMBaseClient) -> None:
+        spec = self.to_ocm_spec()
+        self.id = create_product(ocm, spec)
+
+    def update(self, ocm: OCMBaseClient) -> None:
+        err_msg = "Called update on StatusBoardHandler that doesn't have update method"
+        logging.error(err_msg)
+        raise UpdateNotSupported(err_msg)
+
+    def delete(self, ocm: OCMBaseClient) -> None:
+        if not self.id:
+            logging.error(f'Trying to delete Product "{self.name}" without id')
+            return
+        delete_product(ocm, self.id)
+
+    def summarize(self) -> str:
+        return f'Product: "{self.name}"'
+
+    def to_ocm_spec(self) -> BaseOCMSpec:
+        return {
+            "name": self.name,
+            "fullname": self.fullname,
+        }
+
+    @staticmethod
+    def get_priority() -> int:
+        return 0
+
+
+class Application(AbstractStatusBoard):
+    product: Optional["Product"]
+    services: list["Service"] | None
+
+    def create(self, ocm: OCMBaseClient) -> None:
+        if self.product and self.product.id:
+            spec = self.to_ocm_spec()
+            self.id = create_application(ocm, spec)
+        else:
+            logging.warning("Missing product id for application")
+
+    def update(self, ocm: OCMBaseClient) -> None:
+        err_msg = "Called update on StatusBoardHandler that doesn't have update method"
+        logging.error(err_msg)
+        raise UpdateNotSupported(err_msg)
+
+    def delete(self, ocm: OCMBaseClient) -> None:
+        if not self.id:
+            logging.error(f'Trying to delete Application "{self.name}" without id')
+            return
+        delete_application(ocm, self.id)
+
+    def summarize(self) -> str:
+        return f'Application: "{self.name}" "{self.fullname}"'
+
+    def to_ocm_spec(self) -> ApplicationOCMSpec:
+        product_id = self.product.id if self.product and self.product.id else ""
+        return {
+            "name": self.name,
+            "fullname": self.fullname,
+            "product_id": product_id,
+        }
+
+    @staticmethod
+    def get_priority() -> int:
+        return 1
+
+
+class Service(AbstractStatusBoard):
+    # `application` here is used to create a flat map to easily compare state.
+    # This field is optional so we can create the Service object without the
+    # need to create an Application object first.
+    # This filed is needed when we are creating a Service on teh OCM API.
+    # This field is not used when we are mapping the services that belongs to an
+    # application in that case we use the `services` field in Application class.
+    application: Optional["Application"]
+    metadata: ServiceMetadataSpec
+
+    def create(self, ocm: OCMBaseClient) -> None:
+        spec = self.to_ocm_spec()
+        if self.application and self.application.id:
+            self.id = create_service(ocm, spec)
+        else:
+            logging.warning("Missing application id for service")
+
+    def delete(self, ocm: OCMBaseClient) -> None:
+        if not self.id:
+            logging.error(f'Trying to delete Service "{self.name}" without id')
+            return
+        delete_service(ocm, self.id)
+
+    def update(self, ocm: OCMBaseClient) -> None:
+        if not self.id:
+            logging.error(f'Trying to update Service "{self.name}" without id')
+            return
+        spec = self.to_ocm_spec()
+        if self.application and self.application.id:
+            update_service(ocm, self.id, spec)
+        else:
+            logging.warning("Missing application id for service")
+
+    def summarize(self) -> str:
+        return f'Service: "{self.name}" "{self.fullname}"'
+
+    def to_ocm_spec(self) -> ServiceOCMSpec:
+        application_id = (
+            self.application.id if self.application and self.application.id else ""
+        )
+
+        return {
+            "name": self.name,
+            "fullname": self.fullname,
+            "metadata": self.metadata,
+            "status_type": "traffic_light",
+            "service_endpoint": "none",
+            "application_id": application_id,
+        }
+
+    @staticmethod
+    def get_priority() -> int:
+        return 2
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Service):
+            return NotImplemented
+        return (
+            self.name == other.name
+            and self.fullname == other.fullname
+            and self.metadata == other.metadata
+        )
+
+
+# Resolve forward references after class definitions
+Product.update_forward_refs()
+Application.update_forward_refs()
+Service.update_forward_refs()
+
+
+class UpdateNotSupported(Exception):
+    pass
+
+
+class StatusBoardHandler(BaseModel):
+    action: Action
+    status_board_object: AbstractStatusBoard
+
+    def act(self, dry_run: bool, ocm: OCMBaseClient) -> None:
+        logging.info(f"{self.action} - {self.status_board_object.summarize()}")
+
+        if dry_run:
+            return
+
+        match self.action:
+            case Action.delete:
+                self.status_board_object.delete(ocm)
+            case Action.create:
+                self.status_board_object.create(ocm)
+            case Action.update:
+                self.status_board_object.update(ocm)
+
+
+class StatusBoardExporterIntegration(QontractReconcileIntegration):
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    @staticmethod
+    def get_product_apps(sb: StatusBoardV1) -> dict[str, set[str]]:
+        global_selectors = (
+            sb.global_app_selectors.exclude or [] if sb.global_app_selectors else []
+        )
+        return {
+            p.product_environment.product.name: get_selected_app_names(
+                global_selectors, p
+            )
+            for p in sb.products
+        }
+
+    @staticmethod
+    def get_current_products_applications_services(
+        ocm_api: OCMBaseClient,
+    ) -> list[Product]:
+        products_raw = get_managed_products(ocm_api)
+        products = [Product(**p) for p in products_raw]
+
+        for p in products:
+            if not p.id:
+                logging.error(f'Product "{p.name}" has no id')
+                continue
+            p.applications = [
+                Application(**a) for a in get_product_applications(ocm_api, p.id)
+            ]
+            for a in p.applications:
+                if not a.id:
+                    logging.error(f'Application "{a.name}" has no id')
+                    continue
+                a.services = [
+                    Service(**s) for s in get_application_services(ocm_api, a.id)
+                ]
+
+        return products
+
+    @staticmethod
+    def desired_abstract_status_board_map(
+        desired_product_apps: Mapping[str, set[str]], slodocs: list[SLODocumentV1]
+    ) -> dict[str, AbstractStatusBoard]:
+        """
+        Returns a Mapping of all the AbstractStatusBoard data objects as dictionaries.
+        The key is formed by combining the Product, Application and Service name
+        separeted by a '/' character. This is the same format as the fullname property
+        on Status Board OCM API.
+        """
+        desired_abstract_status_board_map: dict[str, AbstractStatusBoard] = {}
+        for product, apps in desired_product_apps.items():
+            desired_abstract_status_board_map[product] = Product(
+                name=product, fullname=product, applications=[], metadata={}
+            )
+            for a in apps:
+                key = f"{product}/{a}"
+                desired_abstract_status_board_map[key] = Application(
+                    name=a,
+                    fullname=key,
+                    services=[],
+                    product=desired_abstract_status_board_map[product],
+                    metadata={},
+                )
+        for slodoc in slodocs:
+            products = [
+                ns.namespace.environment.product.name for ns in slodoc.namespaces
+            ]
+            for slo in slodoc.slos or []:
+                for product in products:
+                    if slodoc.app.parent_app:
+                        app = f"{slodoc.app.parent_app.name}-{slodoc.app.name}"
+                    else:
+                        app = slodoc.app.name
+
+                    # Check if the product or app is excluded from the desired list
+                    product_or_app_excluded = (
+                        product not in desired_product_apps
+                        or app not in desired_product_apps.get(product, set())
+                    )
+
+                    # Check if statusBoard label exists and is explicitly disabled
+                    status_board_enabled = (
+                        slodoc.labels is not None
+                        and "statusBoard" in slodoc.labels
+                        and slodoc.labels["statusBoard"] == "enabled"
+                    )
+
+                    if product_or_app_excluded or not status_board_enabled:
+                        continue
+
+                    key = f"{product}/{app}/{slo.name}"
+                    metadata = {
+                        "sli_type": slo.sli_type,
+                        "sli_specification": slo.sli_specification,
+                        "slo_details": slo.slo_details,
+                        "target": slo.slo_target,
+                        "target_unit": slo.slo_target_unit,
+                        "window": slo.slo_parameters.window,
+                    }
+                    desired_abstract_status_board_map[key] = Service(
+                        name=slo.name,
+                        fullname=key,
+                        metadata=metadata,
+                        application=desired_abstract_status_board_map[
+                            f"{product}/{app}"
+                        ],
+                    )
+
+        return desired_abstract_status_board_map
+
+    @staticmethod
+    def current_abstract_status_board_map(
+        current_products_applications_services: Iterable[Product],
+    ) -> dict[str, AbstractStatusBoard]:
+        return_value: dict[str, AbstractStatusBoard] = {}
+        for product in current_products_applications_services:
+            return_value[product.name] = product
+            for app in product.applications or []:
+                return_value[f"{product.name}/{app.name}"] = app
+                for service in app.services or []:
+                    return_value[f"{product.name}/{app.name}/{service.name}"] = service
+
+        return return_value
+
+    @staticmethod
+    def get_diff(
+        desired_abstract_status_board_map: Mapping[str, AbstractStatusBoard],
+        current_abstract_status_board_map: Mapping[str, AbstractStatusBoard],
+        current_products: Mapping[str, Product],
+    ) -> list[StatusBoardHandler]:
+        return_list: list[StatusBoardHandler] = []
+
+        diff_result = diff_mappings(
+            current_abstract_status_board_map,
+            desired_abstract_status_board_map,
+        )
+
+        return_list.extend(
+            StatusBoardHandler(action=Action.create, status_board_object=o)
+            for o in diff_result.add.values()
+        )
+
+        return_list.extend(
+            StatusBoardHandler(action=Action.delete, status_board_object=o)
+            for o in diff_result.delete.values()
+        )
+
+        services_to_update = [
+            s.desired
+            for _, s in diff_result.change.items()
+            if isinstance(s.desired, Service)
+        ]
+
+        return_list.extend(
+            StatusBoardHandler(action=Action.update, status_board_object=s)
+            for s in services_to_update
+        )
+
+        return return_list
+
+    @staticmethod
+    def apply_diff(
+        dry_run: bool, ocm_api: OCMBaseClient, diff: list[StatusBoardHandler]
+    ) -> None:
+        creations: list[StatusBoardHandler] = []
+        deletions: list[StatusBoardHandler] = []
+        updates: list[StatusBoardHandler] = []
+
+        for o in diff:
+            match o.action:
+                case Action.create:
+                    creations.append(o)
+                case Action.delete:
+                    deletions.append(o)
+                case Action.update:
+                    updates.append(o)
+
+        # Products need to be created before Applications
+        # Applications need to be created before Services
+        creations.sort(key=lambda x: x.status_board_object.get_priority())
+
+        # Services need to be deleted before Applications
+        # Applications need to be deleted before Products
+        deletions.sort(key=lambda x: x.status_board_object.get_priority(), reverse=True)
+
+        for d in creations + deletions + updates:
+            d.act(dry_run, ocm_api)
+
+    def run(self, dry_run: bool) -> None:
+        slodocs = get_slo_documents()
+        for sb in get_status_board():
+            ocm_api = init_ocm_base_client(sb.ocm, self.secret_reader)
+
+            # Desired state
+            desired_product_apps: dict[str, set[str]] = self.get_product_apps(sb)
+            desired_abstract_status_board_map = self.desired_abstract_status_board_map(
+                desired_product_apps, slodocs
+            )
+
+            # Current state
+            current_products_applications_services = (
+                self.get_current_products_applications_services(ocm_api)
+            )
+            current_abstract_status_board_map = self.current_abstract_status_board_map(
+                current_products_applications_services
+            )
+
+            current_products = {
+                p.name: p for p in current_products_applications_services
+            }
+            diff = self.get_diff(
+                desired_abstract_status_board_map,
+                current_abstract_status_board_map,
+                current_products,
+            )
+
+            self.apply_diff(dry_run, ocm_api, diff)

qontract_reconcile-0.10.2.dev228/reconcile/utils/ocm/status_board.py

@@ -0,0 +1,138 @@
+from typing import Any, TypedDict
+
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+SERVICE_DESIRED_KEYS = {"id", "name", "fullname", "metadata"}
+APPLICATION_DESIRED_KEYS = {"id", "name", "fullname", "metadata"}
+PRODUCTS_DESIRED_KEYS = {"id", "name", "fullname", "metadata"}
+
+
+METADATA_MANAGED_BY_KEY = "managedBy"
+METADATA_MANAGED_BY_VALUE = "qontract-reconcile"
+
+
+class BaseOCMSpec(TypedDict):
+    name: str
+    fullname: str
+
+
+class ApplicationOCMSpec(BaseOCMSpec):
+    product_id: str
+
+
+class ServiceMetadataSpec(TypedDict):
+    sli_type: str
+    sli_specification: str
+    slo_details: str
+    target: float
+    target_unit: str
+    window: str
+
+
+class ServiceOCMSpec(BaseOCMSpec):
+    # The next two fields come from the orignal script at
+    # https://gitlab.cee.redhat.com/service/status-board/-/blob/main/scripts/create-services-from-app-intf.sh?ref_type=heads#L116
+    status_type: str
+    service_endpoint: str
+    application_id: str
+    metadata: ServiceMetadataSpec
+
+
+def get_product_applications(
+    ocm_api: OCMBaseClient, product_id: str
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for application in ocm_api.get_paginated(
+        f"/api/status-board/v1/products/{product_id}/applications"
+    ):
+        if (
+            application.get("metadata", {}).get(METADATA_MANAGED_BY_KEY, "")
+            == METADATA_MANAGED_BY_VALUE
+        ):
+            results.append({  # noqa: PERF401
+                k: v for k, v in application.items() if k in APPLICATION_DESIRED_KEYS
+            })
+
+    return results
+
+
+def get_application_services(
+    ocm_api: OCMBaseClient, app_id: str
+) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+
+    for service in ocm_api.get_paginated(
+        f"/api/status-board/v1/applications/{app_id}/services"
+    ):
+        if (
+            service.get("metadata", {}).get(METADATA_MANAGED_BY_KEY, "")
+            == METADATA_MANAGED_BY_VALUE
+        ):
+            results.append({  # noqa: PERF401
+                k: v for k, v in service.items() if k in SERVICE_DESIRED_KEYS
+            })
+
+    return results
+
+
+def get_managed_products(ocm_api: OCMBaseClient) -> list[dict[str, Any]]:
+    results: list[dict[str, Any]] = []
+    for product in ocm_api.get_paginated("/api/status-board/v1/products/"):
+        if (
+            product.get("metadata", {}).get(METADATA_MANAGED_BY_KEY, "")
+            == METADATA_MANAGED_BY_VALUE
+        ):
+            results.append({  # noqa: PERF401
+                k: v for k, v in product.items() if k in PRODUCTS_DESIRED_KEYS
+            })
+    return results
+
+
+def create_product(ocm_api: OCMBaseClient, spec: BaseOCMSpec) -> str:
+    data = spec | {"metadata": {METADATA_MANAGED_BY_KEY: METADATA_MANAGED_BY_VALUE}}
+
+    resp = ocm_api.post("/api/status-board/v1/products/", data=data)
+    return resp["id"]
+
+
+def create_application(ocm_api: OCMBaseClient, spec: ApplicationOCMSpec) -> str:
+    data = spec | {"metadata": {METADATA_MANAGED_BY_KEY: METADATA_MANAGED_BY_VALUE}}
+
+    resp = ocm_api.post("/api/status-board/v1/applications/", data=data)
+    return resp["id"]
+
+
+def create_service(ocm_api: OCMBaseClient, spec: ServiceOCMSpec) -> str:
+    data = spec | {
+        "metadata": spec["metadata"]
+        | {METADATA_MANAGED_BY_KEY: METADATA_MANAGED_BY_VALUE}
+    }
+
+    resp = ocm_api.post("/api/status-board/v1/services/", data=data)
+    return resp["id"]
+
+
+def update_service(
+    ocm_api: OCMBaseClient,
+    service_id: str,
+    spec: ServiceOCMSpec,
+) -> None:
+    data = spec | {
+        "metadata": spec["metadata"]
+        | {METADATA_MANAGED_BY_KEY: METADATA_MANAGED_BY_VALUE}
+    }
+
+    ocm_api.patch(f"/api/status-board/v1/services/{service_id}", data=data)
+
+
+def delete_product(ocm_api: OCMBaseClient, product_id: str) -> None:
+    ocm_api.delete(f"/api/status-board/v1/products/{product_id}")
+
+
+def delete_application(ocm_api: OCMBaseClient, application_id: str) -> None:
+    ocm_api.delete(f"/api/status-board/v1/applications/{application_id}")
+
+
+def delete_service(ocm_api: OCMBaseClient, service_id: str) -> None:
+    ocm_api.delete(f"/api/status-board/v1/services/{service_id}")