qontract-reconcile 0.10.2.dev179__tar.gz → 0.10.2.dev181__tar.gz

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev179
+Version: 0.10.2.dev181
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/aus/base.py

@@ -35,6 +35,7 @@ from reconcile.aus.metrics import (
     UPGRADE_SCHEDULED_METRIC_VALUE,
     UPGRADE_STARTED_METRIC_VALUE,
     AUSClusterHealthStateGauge,
+    AUSClusterMissingVersionGateAgreementsGauge,
     AUSClusterUpgradePolicyInfoMetric,
     AUSOCMEnvironmentError,
     AUSOrganizationErrorRate,
@@ -1066,6 +1067,7 @@ def calculate_diff(
     ocm_api: OCMBaseClient,
     version_data: VersionData,
     addon_id: str = "",
+    integration: str = "",
 ) -> list[UpgradePolicyHandler]:
     """Check available upgrades for each cluster in the desired state
     according to upgrade conditions
@@ -1169,13 +1171,25 @@ def calculate_diff(
                     },
                 )
                 if gates_with_missing_agreements:
-                    missing_gate_ids = [
-                        gate.id for gate in gates_with_missing_agreements
+                    missing_gate_labels = [
+                        gate.label for gate in gates_with_missing_agreements
                     ]
                     logging.info(
-                        f"[{spec.org.org_id}/{spec.org.name}/{spec.cluster.name}] found gates with missing agreements for {target_version_prefix} - {missing_gate_ids} "
+                        f"[{spec.org.org_id}/{spec.org.name}/{spec.cluster.name}] found gates with missing agreements for {target_version_prefix} - {missing_gate_labels} "
                         "Skip creation of an upgrade policy until all of them have been acked by the version-gate-approver integration or a user."
                     )
+
+                    metrics.set_gauge(
+                        AUSClusterMissingVersionGateAgreementsGauge(
+                            integration=integration,
+                            ocm_env=spec.org.environment.name,
+                            org_id=spec.org.org_id,
+                            cluster_uuid=spec.cluster.id,
+                            version_prefix=target_version_prefix,
+                        ),
+                        len(gates_with_missing_agreements),
+                    )
+
                     continue
                 diffs.append(
                     UpgradePolicyHandler(

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/aus/metrics.py

@@ -56,6 +56,18 @@ class AUSClusterHealthStateGauge(AUSBaseMetric, GaugeMetric):
         return "aus_cluster_health_state"
 
 
+class AUSClusterMissingVersionGateAgreementsGauge(AUSBaseMetric, GaugeMetric):
+    "The number of missing version gate agreements for a cluster and a target version prefix."
+
+    org_id: str
+    cluster_uuid: str
+    version_prefix: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_cluster_missing_version_gate_agreements"
+
+
 class AUSAddonVersionRemainingSoakDaysGauge(AUSClusterVersionRemainingSoakDaysGauge):
     "Remaining days a version needs to soak for an addon on a cluster"
 

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/aus/ocm_addons_upgrade_scheduler_org.py

@@ -250,6 +250,7 @@ def calculate_diff(
         ocm_api,
         version_data,
         addon_id,
+        integration=QONTRACT_INTEGRATION,
     )
     for current in addon_current_state:
         if addon_id == current.addon_id and (

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/aus/ocm_upgrade_scheduler.py

@@ -76,7 +76,11 @@ class OCMClusterUpgradeSchedulerIntegration(
             )
 
             diffs = aus.calculate_diff(
-                current_state, org_upgrade_spec, ocm_api, version_data
+                current_state,
+                org_upgrade_spec,
+                ocm_api,
+                version_data,
+                integration=self.name,
             )
             aus.act(dry_run, diffs, ocm_api)
 

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/saas_auto_promotions_manager/integration.py

@@ -94,6 +94,7 @@ class SaasAutoPromotionsManager:
 
 def init_external_dependencies(
     dry_run: bool,
+    thread_pool_size: int,
     env_name: str | None = None,
     app_name: str | None = None,
 ) -> tuple[
@@ -139,7 +140,11 @@ def init_external_dependencies(
         renderer=Renderer(),
     )
     saas_files = get_saas_files(env_name=env_name, app_name=app_name)
-    saas_inventory = SaasFilesInventory(saas_files=saas_files)
+    saas_inventory = SaasFilesInventory(
+        saas_files=saas_files,
+        secret_reader=secret_reader,
+        thread_pool_size=thread_pool_size,
+    )
     saas_deploy_state = init_state(
         integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
     )
@@ -181,7 +186,10 @@ def run(
         saas_deploy_state,
         sapm_state,
     ) = init_external_dependencies(
-        dry_run=dry_run, env_name=env_name, app_name=app_name
+        dry_run=dry_run,
+        env_name=env_name,
+        app_name=app_name,
+        thread_pool_size=thread_pool_size,
     )
     if defer:
         defer(vcs.cleanup)

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/saas_auto_promotions_manager/subscriber.py

@@ -13,6 +13,7 @@ from reconcile.saas_auto_promotions_manager.publisher import (
     DeploymentInfo,
     Publisher,
 )
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 CONTENT_HASH_LENGTH = 32
 
@@ -47,7 +48,9 @@ class Subscriber:
         uid: str,
         soak_days: int,
         blocked_versions: set[str],
+        hotfix_versions: set[str],
         schedule: str,
+        slo_document_manager: SLODocumentManager | None = None,
     ):
         self.saas_name = saas_name
         self.template_name = template_name
@@ -64,6 +67,8 @@ class Subscriber:
         self._content_hash = ""
         self._use_target_config_hash = use_target_config_hash
         self._blocked_versions = blocked_versions
+        self._hotfix_versions = hotfix_versions
+        self.slo_document_manager = slo_document_manager
 
     def has_diff(self) -> bool:
         current_hashes = {
@@ -192,6 +197,10 @@ class Subscriber:
             return
 
         desired_ref = next(iter(publisher_refs))
+        # validate slo gatekeeping
+        if self._has_breached_slos(desired_ref=desired_ref):
+            return
+
         if desired_ref in self._blocked_versions:
             logging.info(
                 "Subscriber at path %s promotion stopped because of blocked ref: %s",
@@ -203,6 +212,25 @@ class Subscriber:
         # Passed all gates -> lets promote desired ref
         self.desired_ref = desired_ref
 
+    def _has_breached_slos(self, desired_ref: str) -> bool:
+        if self.slo_document_manager and desired_ref not in self._hotfix_versions:
+            breached_slos = self.slo_document_manager.get_breached_slos()
+            if breached_slos:
+                logging.info(
+                    "Subscriber at path %s promotion stopped because following breached SLOs",
+                    self.target_file_path,
+                )
+                for slo in breached_slos:
+                    logging.info(
+                        "SLO:%s of document %s is breached. Current value:%f Expected:%f ",
+                        slo.slo.name,
+                        slo.slo_document_name,
+                        slo.current_slo_value,
+                        slo.slo.slo_target,
+                    )
+                return True
+        return False
+
     def _compute_desired_config_hashes(self) -> None:
         """
         Compute the desired config hashes for this subscriber.

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from collections.abc import Iterable
 
 from reconcile.gql_definitions.common.saas_files import ParentSaasPromotionV1
@@ -8,7 +9,9 @@ from reconcile.saas_auto_promotions_manager.subscriber import (
     ConfigHash,
     Subscriber,
 )
-from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.typed_queries.saas_files import SaasFile, SaasResourceTemplateTarget
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 
 class SaasFileInventoryError(Exception):
@@ -25,8 +28,15 @@ class SaasFilesInventory:
     This basically spans a directed graph, with subscribers as the root.
     """
 
-    def __init__(self, saas_files: Iterable[SaasFile]):
+    def __init__(
+        self,
+        saas_files: Iterable[SaasFile],
+        secret_reader: SecretReaderBase,
+        thread_pool_size: int,
+    ):
         self._saas_files = saas_files
+        self.secret_reader = secret_reader
+        self.thread_pool_size = thread_pool_size
         self._channels_by_name: dict[str, Channel] = {}
         self.subscribers: list[Subscriber] = []
         self.publishers: list[Publisher] = []
@@ -86,10 +96,13 @@ class SaasFilesInventory:
 
     def _assemble_subscribers_with_auto_promotions(self) -> None:
         for saas_file in self._saas_files:
-            blocked_versions: dict[str, set[str]] = {}
+            blocked_versions: dict[str, set[str]] = defaultdict(set[str])
+            hotfix_versions: dict[str, set[str]] = defaultdict(set[str])
             for code_component in saas_file.app.code_components or []:
                 for version in code_component.blocked_versions or []:
-                    blocked_versions.setdefault(code_component.url, set()).add(version)
+                    blocked_versions[code_component.url].add(version)
+                for hf_version in code_component.hotfix_versions or []:
+                    hotfix_versions[code_component.url].add(hf_version)
             for resource_template in saas_file.resource_templates:
                 for target in resource_template.targets:
                     file_path = target.path or saas_file.path
@@ -101,6 +114,7 @@ class SaasFilesInventory:
                         continue
                     soak_days = target.promotion.soak_days or 0
                     schedule = target.promotion.schedule or "* * * * *"
+
                     subscriber = Subscriber(
                         uid=target.uid(
                             parent_saas_file_name=saas_file.name,
@@ -112,10 +126,10 @@ class SaasFilesInventory:
                         ref=target.ref,
                         target_namespace=target.namespace,
                         soak_days=soak_days,
+                        slo_document_manager=self._build_slo_document_manager(target),
                         schedule=schedule,
-                        blocked_versions=blocked_versions.get(
-                            resource_template.url, set()
-                        ),
+                        hotfix_versions=hotfix_versions[resource_template.url],
+                        blocked_versions=blocked_versions[resource_template.url],
                         use_target_config_hash=bool(
                             target.promotion.redeploy_on_publisher_config_change
                         ),
@@ -158,6 +172,17 @@ class SaasFilesInventory:
                             self._channels_by_name[subscribe_channel]
                         )
 
+    def _build_slo_document_manager(
+        self, target: SaasResourceTemplateTarget
+    ) -> SLODocumentManager | None:
+        if target.slos:
+            return SLODocumentManager(
+                slo_documents=target.slos,
+                secret_reader=self.secret_reader,
+                thread_pool_size=self.thread_pool_size,
+            )
+        return None
+
     def _remove_unsupported(self) -> None:
         """
         Lets remove subscribers from which we know we do not support them and log an error.

{qontract_reconcile-0.10.2.dev179 → qontract_reconcile-0.10.2.dev181}/reconcile/typed_queries/saas_files.py

@@ -39,6 +39,7 @@ from reconcile.gql_definitions.common.saasherder_settings import AppInterfaceSet
 from reconcile.gql_definitions.common.saasherder_settings import (
     query as saasherder_settings_query,
 )
+from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
 from reconcile.gql_definitions.fragments.saas_target_namespace import (
     SaasTargetNamespace,
 )
@@ -63,6 +64,7 @@ class SaasResourceTemplateTarget(ConfiguredBaseModel):
     secret_parameters: (
         list[SaasResourceTemplateTargetV2_SaasSecretParametersV1] | None
     ) = Field(..., alias="secretParameters")
+    slos: list[SLODocument] | None = Field(..., alias="slos")
     upstream: SaasResourceTemplateTargetUpstreamV1 | None = Field(..., alias="upstream")
     images: list[SaasResourceTemplateTargetImageV1] | None = Field(..., alias="images")
     disable: bool | None = Field(..., alias="disable")