qontract-reconcile 0.10.2.dev179__py3-none-any.whl → 0.10.2.dev181__py3-none-any.whl

{qontract_reconcile-0.10.2.dev179.dist-info → qontract_reconcile-0.10.2.dev181.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev179
+Version: 0.10.2.dev181
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev179.dist-info → qontract_reconcile-0.10.2.dev181.dist-info}/RECORD

@@ -123,14 +123,14 @@ reconcile/vpc_peerings_validator.py,sha256=aESqrhm1tpkc2iqSL1UV5to_HjNgjRSffD0cr
 reconcile/aus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aus/advanced_upgrade_service.py,sha256=lt684trHbKvVDLwwuNVz3Wu_MnytFSbS_7MZTIITh9k,23969
 reconcile/aus/aus_label_source.py,sha256=o0S2f0qwcII_8nzhHZhRQ83gEZ1DrSXyO4xzSwLebuU,4382
-reconcile/aus/base.py,sha256=rx2OuShoFRP7O6Kov9rRjEkhCPpPavfDF81tieB6XFg,50747
+reconcile/aus/base.py,sha256=Kicj39qfAlINywfk0Legu7RUcUxI4AuoBhA8vJ1gWag,51343
 reconcile/aus/cluster_version_data.py,sha256=VZWbUEIbrDKO-sroMpQtiWCTqDraTMd8tssKV0HyTQ0,7140
 reconcile/aus/healthchecks.py,sha256=jR9c-syh9impnkV0fd6XW3Bnk7iRN5zv8oCRYM-yIRY,2700
-reconcile/aus/metrics.py,sha256=nKT4m2zGT-QOMR0c-z-npVNKWsNMubzdffpU_f9n4II,3927
+reconcile/aus/metrics.py,sha256=qh3-oWL8-Hbj1uXgAKonw7sVXGBlPpCyHtVYJusZ9n8,4271
 reconcile/aus/models.py,sha256=qLjWLDJe5PGXPPtJ5PI01IVEYaSGweu9dkAgf0ZM2hk,8297
 reconcile/aus/node_pool_spec.py,sha256=FkMggklG-4BgQwud2Swp2m3AAAKzZmeaXgohl9uwxZ8,1138
-reconcile/aus/ocm_addons_upgrade_scheduler_org.py,sha256=SHbFEEq9ETtkkuzBBaNuwSDze8fj6DWHeA_XNgOyTbo,10308
-reconcile/aus/ocm_upgrade_scheduler.py,sha256=2uPn13y3QGCHLoKwCc1Z7q9wQsoQf_F1HATMYUbl53s,3695
+reconcile/aus/ocm_addons_upgrade_scheduler_org.py,sha256=PL8QdzWh8lhGneMwSbIYxOp002mJd5gHf_T0Q2cWQho,10350
+reconcile/aus/ocm_upgrade_scheduler.py,sha256=WPIUUr3n3j-ZtyE3DgsPKzBJ3Dm9i1PSHBRepAf6CF4,3783
 reconcile/aus/ocm_upgrade_scheduler_org.py,sha256=QeZAQ1wdDPcwZ77b3Xmt4yBV60rWi3qd8h-vGwnwoCs,3948
 reconcile/aus/upgrades.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aus/version_gate_approver.py,sha256=iZg7l-VgcgyVf7jV9mLZzXq3d22r8feAnl-1LDVUJR4,7358
@@ -482,11 +482,11 @@ reconcile/rhidp/sso_client/base.py,sha256=EfQ2ewcOKh5idg46UKAkY6z0m_nGQfvnQKffa2
 reconcile/rhidp/sso_client/integration.py,sha256=kA8g7c38ZBSdrRtyfEqy_WgSreD1PbwY7ZIN-3tZRPc,2221
 reconcile/rhidp/sso_client/metrics.py,sha256=Tq7tSOsqL3XdcPUdozxqzSPIodUeOV87UCTqpuuqqhw,1013
 reconcile/saas_auto_promotions_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/saas_auto_promotions_manager/integration.py,sha256=haXTpwi0rEQtN9-MadBzFvSNtbxw5QQuFpMQLg3bZlQ,6707
+reconcile/saas_auto_promotions_manager/integration.py,sha256=PhngUNVQMdZ_7KlIOCokdYqQ2k-XdsvvfCb8BAdSiAA,6889
 reconcile/saas_auto_promotions_manager/meta.py,sha256=76Jp50r6Y_KyJoXFfSjrt5YrCtXyg_A4FXXxHYiS3TE,161
 reconcile/saas_auto_promotions_manager/publisher.py,sha256=5gphMxr2NUvyB7WDK4eAbgZeyeF30cZ3a2ZGrbFQgZk,2976
 reconcile/saas_auto_promotions_manager/s3_exporter.py,sha256=Y-r5R6viiAzglUHbYKItYSjT_axmLlPEJVmu_H6N170,2682
-reconcile/saas_auto_promotions_manager/subscriber.py,sha256=tZLcjPt64cIxiswlSGpEcAg3VTp0PHvnDqgFQ2pajVM,10044
+reconcile/saas_auto_promotions_manager/subscriber.py,sha256=V8e2tz0s3A-pcUNb2UoREQm6-6ZzS4F3gJdGUBoa9WY,11281
 reconcile/saas_auto_promotions_manager/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/saas_auto_promotions_manager/merge_request_manager/batcher.py,sha256=R2CRtjdOggY5lSqt-A5qz2ymqomx4opVeVV_oqBAW0A,7804
 reconcile/saas_auto_promotions_manager/merge_request_manager/desired_state.py,sha256=isY8frVsL3PlcdZmdZ4O0qyp76oczl4DUMX9uMArs5Y,1222
@@ -497,7 +497,7 @@ reconcile/saas_auto_promotions_manager/merge_request_manager/mr_parser.py,sha256
 reconcile/saas_auto_promotions_manager/merge_request_manager/open_merge_requests.py,sha256=-qGQOh6Jdp4lomNDij3zWVC0pl6uPHFWS5Woqcp5HQk,410
 reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py,sha256=EgImn9SpeThTNN5-P5lvz__bMKZNU4m2SfKpocRCy5w,7278
 reconcile/saas_auto_promotions_manager/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py,sha256=LhK3OtHf-l_wm1SZCb3vZUrPpNZJRpM65WKY2HATXRY,8606
+reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py,sha256=BPzSsuXy5ULKSo36v1bItC3rU-AEFfVZeWXZw1tXqLg,9674
 reconcile/skupper_network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/skupper_network/integration.py,sha256=oZIVDBRcQPC-lWxNFiJhGbtCM7Yj7fjwAzYZ8JvVe3I,10789
 reconcile/skupper_network/models.py,sha256=HEwlVKsbmMaKaaBGvITIiSYNEVdjwXVhLaOJgLSZ2xQ,6604
@@ -569,7 +569,7 @@ reconcile/typed_queries/pagerduty_instances.py,sha256=zxCNxMak4iikryePaRi71lTADV
 reconcile/typed_queries/quay.py,sha256=3IMy9jjHF2f9t47EXZOQVA3p0nFkWFhaFhxhvib-71o,644
 reconcile/typed_queries/repos.py,sha256=8A93dKDt6igT4ClqMjt7YUTsoP4qh1Wnm0W3xsMgj48,824
 reconcile/typed_queries/reserved_networks.py,sha256=XY9y3amtIQT0n06O0Toubqr_UmylJ2ELAv9-BJCK890,345
-reconcile/typed_queries/saas_files.py,sha256=O2kd0nSFfMgnbXvSv9oMIdlBGZg7XlOU3y2CWg1W2DQ,14001
+reconcile/typed_queries/saas_files.py,sha256=SOE36sWPBcuaRmEaNxXCQZMQdJiUZX8_A92o42XwHQA,14141
 reconcile/typed_queries/slack.py,sha256=r30lspctHloyygPn8_DVybxPwUWwiBpvBRRXiTVcQYk,251
 reconcile/typed_queries/slo_documents.py,sha256=YMdox_-lBRqrdxamPhdnUlRTY_Ro35ptsupq7OaynUQ,362
 reconcile/typed_queries/smtp.py,sha256=aSLglYa5bHKmlGwKkxq2RZqyMWuAf0a4S_mOuhDa084,542
@@ -807,7 +807,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev179.dist-info/METADATA,sha256=vGUEKBGnRX-V-Y5trKn107wjTWzK8k9k_cbQ92eYeLE,24627
-qontract_reconcile-0.10.2.dev179.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev179.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev179.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev181.dist-info/METADATA,sha256=GFcwt6xmX1m6ffKhsTJe6HD5YUe_VT-8-OiGbWIN2cE,24627
+qontract_reconcile-0.10.2.dev181.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev181.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev181.dist-info/RECORD,,

reconcile/aus/base.py

@@ -35,6 +35,7 @@ from reconcile.aus.metrics import (
     UPGRADE_SCHEDULED_METRIC_VALUE,
     UPGRADE_STARTED_METRIC_VALUE,
     AUSClusterHealthStateGauge,
+    AUSClusterMissingVersionGateAgreementsGauge,
     AUSClusterUpgradePolicyInfoMetric,
     AUSOCMEnvironmentError,
     AUSOrganizationErrorRate,
@@ -1066,6 +1067,7 @@ def calculate_diff(
     ocm_api: OCMBaseClient,
     version_data: VersionData,
     addon_id: str = "",
+    integration: str = "",
 ) -> list[UpgradePolicyHandler]:
     """Check available upgrades for each cluster in the desired state
     according to upgrade conditions
@@ -1169,13 +1171,25 @@ def calculate_diff(
                     },
                 )
                 if gates_with_missing_agreements:
-                    missing_gate_ids = [
-                        gate.id for gate in gates_with_missing_agreements
+                    missing_gate_labels = [
+                        gate.label for gate in gates_with_missing_agreements
                     ]
                     logging.info(
-                        f"[{spec.org.org_id}/{spec.org.name}/{spec.cluster.name}] found gates with missing agreements for {target_version_prefix} - {missing_gate_ids} "
+                        f"[{spec.org.org_id}/{spec.org.name}/{spec.cluster.name}] found gates with missing agreements for {target_version_prefix} - {missing_gate_labels} "
                         "Skip creation of an upgrade policy until all of them have been acked by the version-gate-approver integration or a user."
                     )
+
+                    metrics.set_gauge(
+                        AUSClusterMissingVersionGateAgreementsGauge(
+                            integration=integration,
+                            ocm_env=spec.org.environment.name,
+                            org_id=spec.org.org_id,
+                            cluster_uuid=spec.cluster.id,
+                            version_prefix=target_version_prefix,
+                        ),
+                        len(gates_with_missing_agreements),
+                    )
+
                     continue
                 diffs.append(
                     UpgradePolicyHandler(

reconcile/aus/metrics.py

@@ -56,6 +56,18 @@ class AUSClusterHealthStateGauge(AUSBaseMetric, GaugeMetric):
         return "aus_cluster_health_state"
 
 
+class AUSClusterMissingVersionGateAgreementsGauge(AUSBaseMetric, GaugeMetric):
+    "The number of missing version gate agreements for a cluster and a target version prefix."
+
+    org_id: str
+    cluster_uuid: str
+    version_prefix: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_cluster_missing_version_gate_agreements"
+
+
 class AUSAddonVersionRemainingSoakDaysGauge(AUSClusterVersionRemainingSoakDaysGauge):
     "Remaining days a version needs to soak for an addon on a cluster"
 

reconcile/aus/ocm_addons_upgrade_scheduler_org.py

@@ -250,6 +250,7 @@ def calculate_diff(
         ocm_api,
         version_data,
         addon_id,
+        integration=QONTRACT_INTEGRATION,
     )
     for current in addon_current_state:
         if addon_id == current.addon_id and (

reconcile/aus/ocm_upgrade_scheduler.py

@@ -76,7 +76,11 @@ class OCMClusterUpgradeSchedulerIntegration(
             )
 
             diffs = aus.calculate_diff(
-                current_state, org_upgrade_spec, ocm_api, version_data
+                current_state,
+                org_upgrade_spec,
+                ocm_api,
+                version_data,
+                integration=self.name,
             )
             aus.act(dry_run, diffs, ocm_api)
 

reconcile/saas_auto_promotions_manager/integration.py

@@ -94,6 +94,7 @@ class SaasAutoPromotionsManager:
 
 def init_external_dependencies(
     dry_run: bool,
+    thread_pool_size: int,
     env_name: str | None = None,
     app_name: str | None = None,
 ) -> tuple[
@@ -139,7 +140,11 @@ def init_external_dependencies(
         renderer=Renderer(),
     )
     saas_files = get_saas_files(env_name=env_name, app_name=app_name)
-    saas_inventory = SaasFilesInventory(saas_files=saas_files)
+    saas_inventory = SaasFilesInventory(
+        saas_files=saas_files,
+        secret_reader=secret_reader,
+        thread_pool_size=thread_pool_size,
+    )
     saas_deploy_state = init_state(
         integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader
     )
@@ -181,7 +186,10 @@ def run(
         saas_deploy_state,
         sapm_state,
     ) = init_external_dependencies(
-        dry_run=dry_run, env_name=env_name, app_name=app_name
+        dry_run=dry_run,
+        env_name=env_name,
+        app_name=app_name,
+        thread_pool_size=thread_pool_size,
     )
     if defer:
         defer(vcs.cleanup)

reconcile/saas_auto_promotions_manager/subscriber.py

@@ -13,6 +13,7 @@ from reconcile.saas_auto_promotions_manager.publisher import (
     DeploymentInfo,
     Publisher,
 )
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 CONTENT_HASH_LENGTH = 32
 
@@ -47,7 +48,9 @@ class Subscriber:
         uid: str,
         soak_days: int,
         blocked_versions: set[str],
+        hotfix_versions: set[str],
         schedule: str,
+        slo_document_manager: SLODocumentManager | None = None,
     ):
         self.saas_name = saas_name
         self.template_name = template_name
@@ -64,6 +67,8 @@ class Subscriber:
         self._content_hash = ""
         self._use_target_config_hash = use_target_config_hash
         self._blocked_versions = blocked_versions
+        self._hotfix_versions = hotfix_versions
+        self.slo_document_manager = slo_document_manager
 
     def has_diff(self) -> bool:
         current_hashes = {
@@ -192,6 +197,10 @@ class Subscriber:
             return
 
         desired_ref = next(iter(publisher_refs))
+        # validate slo gatekeeping
+        if self._has_breached_slos(desired_ref=desired_ref):
+            return
+
         if desired_ref in self._blocked_versions:
             logging.info(
                 "Subscriber at path %s promotion stopped because of blocked ref: %s",
@@ -203,6 +212,25 @@ class Subscriber:
         # Passed all gates -> lets promote desired ref
         self.desired_ref = desired_ref
 
+    def _has_breached_slos(self, desired_ref: str) -> bool:
+        if self.slo_document_manager and desired_ref not in self._hotfix_versions:
+            breached_slos = self.slo_document_manager.get_breached_slos()
+            if breached_slos:
+                logging.info(
+                    "Subscriber at path %s promotion stopped because following breached SLOs",
+                    self.target_file_path,
+                )
+                for slo in breached_slos:
+                    logging.info(
+                        "SLO:%s of document %s is breached. Current value:%f Expected:%f ",
+                        slo.slo.name,
+                        slo.slo_document_name,
+                        slo.current_slo_value,
+                        slo.slo.slo_target,
+                    )
+                return True
+        return False
+
     def _compute_desired_config_hashes(self) -> None:
         """
         Compute the desired config hashes for this subscriber.

reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py

@@ -1,4 +1,5 @@
 import logging
+from collections import defaultdict
 from collections.abc import Iterable
 
 from reconcile.gql_definitions.common.saas_files import ParentSaasPromotionV1
@@ -8,7 +9,9 @@ from reconcile.saas_auto_promotions_manager.subscriber import (
     ConfigHash,
     Subscriber,
 )
-from reconcile.typed_queries.saas_files import SaasFile
+from reconcile.typed_queries.saas_files import SaasFile, SaasResourceTemplateTarget
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.slo_document_manager import SLODocumentManager
 
 
 class SaasFileInventoryError(Exception):
@@ -25,8 +28,15 @@ class SaasFilesInventory:
     This basically spans a directed graph, with subscribers as the root.
     """
 
-    def __init__(self, saas_files: Iterable[SaasFile]):
+    def __init__(
+        self,
+        saas_files: Iterable[SaasFile],
+        secret_reader: SecretReaderBase,
+        thread_pool_size: int,
+    ):
         self._saas_files = saas_files
+        self.secret_reader = secret_reader
+        self.thread_pool_size = thread_pool_size
         self._channels_by_name: dict[str, Channel] = {}
         self.subscribers: list[Subscriber] = []
         self.publishers: list[Publisher] = []
@@ -86,10 +96,13 @@ class SaasFilesInventory:
 
     def _assemble_subscribers_with_auto_promotions(self) -> None:
         for saas_file in self._saas_files:
-            blocked_versions: dict[str, set[str]] = {}
+            blocked_versions: dict[str, set[str]] = defaultdict(set[str])
+            hotfix_versions: dict[str, set[str]] = defaultdict(set[str])
             for code_component in saas_file.app.code_components or []:
                 for version in code_component.blocked_versions or []:
-                    blocked_versions.setdefault(code_component.url, set()).add(version)
+                    blocked_versions[code_component.url].add(version)
+                for hf_version in code_component.hotfix_versions or []:
+                    hotfix_versions[code_component.url].add(hf_version)
             for resource_template in saas_file.resource_templates:
                 for target in resource_template.targets:
                     file_path = target.path or saas_file.path
@@ -101,6 +114,7 @@ class SaasFilesInventory:
                         continue
                     soak_days = target.promotion.soak_days or 0
                     schedule = target.promotion.schedule or "* * * * *"
+
                     subscriber = Subscriber(
                         uid=target.uid(
                             parent_saas_file_name=saas_file.name,
@@ -112,10 +126,10 @@ class SaasFilesInventory:
                         ref=target.ref,
                         target_namespace=target.namespace,
                         soak_days=soak_days,
+                        slo_document_manager=self._build_slo_document_manager(target),
                         schedule=schedule,
-                        blocked_versions=blocked_versions.get(
-                            resource_template.url, set()
-                        ),
+                        hotfix_versions=hotfix_versions[resource_template.url],
+                        blocked_versions=blocked_versions[resource_template.url],
                         use_target_config_hash=bool(
                             target.promotion.redeploy_on_publisher_config_change
                         ),
@@ -158,6 +172,17 @@ class SaasFilesInventory:
                             self._channels_by_name[subscribe_channel]
                         )
 
+    def _build_slo_document_manager(
+        self, target: SaasResourceTemplateTarget
+    ) -> SLODocumentManager | None:
+        if target.slos:
+            return SLODocumentManager(
+                slo_documents=target.slos,
+                secret_reader=self.secret_reader,
+                thread_pool_size=self.thread_pool_size,
+            )
+        return None
+
     def _remove_unsupported(self) -> None:
         """
         Lets remove subscribers from which we know we do not support them and log an error.

reconcile/typed_queries/saas_files.py

@@ -39,6 +39,7 @@ from reconcile.gql_definitions.common.saasherder_settings import AppInterfaceSet
 from reconcile.gql_definitions.common.saasherder_settings import (
     query as saasherder_settings_query,
 )
+from reconcile.gql_definitions.fragments.saas_slo_document import SLODocument
 from reconcile.gql_definitions.fragments.saas_target_namespace import (
     SaasTargetNamespace,
 )
@@ -63,6 +64,7 @@ class SaasResourceTemplateTarget(ConfiguredBaseModel):
     secret_parameters: (
         list[SaasResourceTemplateTargetV2_SaasSecretParametersV1] | None
     ) = Field(..., alias="secretParameters")
+    slos: list[SLODocument] | None = Field(..., alias="slos")
     upstream: SaasResourceTemplateTargetUpstreamV1 | None = Field(..., alias="upstream")
     images: list[SaasResourceTemplateTargetImageV1] | None = Field(..., alias="images")
     disable: bool | None = Field(..., alias="disable")