qontract-reconcile 0.10.1rc923__py3-none-any.whl → 0.10.1rc925__py3-none-any.whl

{qontract_reconcile-0.10.1rc923.dist-info → qontract_reconcile-0.10.1rc925.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc923
+Version: 0.10.1rc925
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc923.dist-info → qontract_reconcile-0.10.1rc925.dist-info}/RECORD

@@ -463,11 +463,11 @@ reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9
 reconcile/templates/rosa-classic-cluster-creation.sh.j2,sha256=FVBmnR2FmVModhqOYNBInhF8zk0Qnj9og9KHK5-X9v0,2361
 reconcile/templates/rosa-hcp-cluster-creation.sh.j2,sha256=2wTdv9qvapCvT8NSi_hq8sXhpFSaxRX-V6Cao1diCI8,2393
 reconcile/templating/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/renderer.py,sha256=oUUiHHSgD5bcUN3Xf-yoEezAf3LjqN-2GlYhQUTPIZY,13067
+reconcile/templating/renderer.py,sha256=GjjYPDmiP0zvTDxOr_vAniqN9UHcSMiv-biPUvy8NzM,13091
 reconcile/templating/validator.py,sha256=5f9f35PCHOOdjb7KZquL2YdabyuAUokPDa4xutSEHIQ,5360
 reconcile/templating/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/templating/lib/merge_request_manager.py,sha256=4oe3EwQOP7CZSraocivbRzzOuVb0ooElaUS2_DGsF50,5603
-reconcile/templating/lib/model.py,sha256=fb6FYYLQjmoh2DjVKO7TEWCuDPf1Q34xmOx0M9Z07ek,324
+reconcile/templating/lib/merge_request_manager.py,sha256=P4IbblzXbubjlaxAYoahHFADUNIXLepfqwzdnHPs8OY,5608
+reconcile/templating/lib/model.py,sha256=Y1PeEP-koHKiH46JSwv4-PGRGQtcxZa9VmjTiuI6Lxs,889
 reconcile/templating/lib/rendering.py,sha256=6kt8NCCwB4vLKYal7KtRmBDguIC1p_PIQCRr-vL7p5w,5504
 reconcile/terraform_init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/terraform_init/integration.py,sha256=glQ9uy8Kj2aTQXCAupwSFeih7reX_xMX_UuWW_ywBMU,6100
@@ -537,7 +537,7 @@ reconcile/test/test_quay_repos.py,sha256=TdkcRF_a8PLp01Kti9eZZN-vGup2yPBT4Iba3k0
 reconcile/test/test_queries.py,sha256=SpH3RmNpBjEr_ne3VjAMCgKK8RE1z1zo7bypkT5uoO4,1946
 reconcile/test/test_repo_owners.py,sha256=uRYMLbMmh-9usF0TerabZTZV-Z1CS4I6ybT-LQqCLe8,1423
 reconcile/test/test_requests_sender.py,sha256=7fd9C2kEFS0-CYtlsif66N1kO9c44pzuBPAJKR9igqU,5385
-reconcile/test/test_saasherder.py,sha256=f_8W8RC5di6IgiqtFaeRi6E9RJ43MGU2zFsLR_p30FQ,54524
+reconcile/test/test_saasherder.py,sha256=jUn8d1DtdXq1pfDrThqIwx37Lgxa7mou3aZ-m3UrWdc,57129
 reconcile/test/test_saasherder_allowed_secret_paths.py,sha256=5NHQwNJO66at6HiyMZ5sVRTQDwxdvlOQo0KmkBWCw5Q,4853
 reconcile/test/test_secret_reader.py,sha256=kz7nzcPjvA08cytnvcA_PMA98AEyqJWsESkYeRn5xCk,4994
 reconcile/test/test_slack_base.py,sha256=pTUGvJ2S2wF3PhJyGWmiNXG52QtXKy2cbu-G8Ymrv6I,5019
@@ -689,7 +689,7 @@ reconcile/utils/pagerduty_api.py,sha256=_24i9S_4X7nlvHb-7clXRE0p1BG4ODjOzKxWO-F9
 reconcile/utils/parse_dhms_duration.py,sha256=TONpLnec5gHeF7k815YNJpQyDjXhkxZIcv9s8ffbTSY,1840
 reconcile/utils/password_validator.py,sha256=XwuWg-8CPlcuG7dl_oQ1G1h2gSVSnfMym_VkuprpWVg,2183
 reconcile/utils/prometheus.py,sha256=Ad0rwLbxRuuYjHwkwJloHEdK0bvy42h-p-HIT1DhDhs,3832
-reconcile/utils/promotion_state.py,sha256=Sd3otIb54vgpEe0JhFG1tkXhPS51LItmkI19pZJhDbE,3607
+reconcile/utils/promotion_state.py,sha256=drkR0PSzfsUVPB0pisARzvUfHgk8KPbeK4FUZdP099Y,3894
 reconcile/utils/promtool.py,sha256=UmBfTHgW9Ys7fZ9BfhIVJEFGLkbge9y1AgL5PNHp7iA,2831
 reconcile/utils/quay_api.py,sha256=zbwi3YjL7dTDYHGWcrZ0mbxyZQuEB8v3sV_Km2O-mIs,7906
 reconcile/utils/raw_github_api.py,sha256=O6Q4vq7bi5ZWcfquPutc9rJ4Ef8_sFqd_RLgzpIoj0w,2920
@@ -793,7 +793,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=C2wrw34OXypshVocAsPrVZsSHptgw4g9u7Haa2wulZQ,9087
 reconcile/utils/saasherder/models.py,sha256=6MGie9SqsyP5ySjmk5bO5vPJ0-x53a0uzABxQO-WsB0,9746
-reconcile/utils/saasherder/saasherder.py,sha256=H3pLQAD6EkjwrFpfQN73LKPg8mYRilssIcZ9M3duizU,84555
+reconcile/utils/saasherder/saasherder.py,sha256=ZTDf6uIpke6y9jKZgESjdY8hDI6Jf2J2UGZp6LJXBWY,85249
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=3gUIIIEv52Vx7-VgQ2FZYfCCrfqUv_5gw_TQ3mbLcTs,4666
@@ -844,8 +844,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc923.dist-info/METADATA,sha256=OjR8Y-8f7JvaPxAt_N-8baMq5aI7LwcbOL2Cirs809I,2273
-qontract_reconcile-0.10.1rc923.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc923.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc923.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc923.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc925.dist-info/METADATA,sha256=JJstPmbQ_muKOcYIt2w0XSiI3q4MlNNDRsJ0enydTfk,2273
+qontract_reconcile-0.10.1rc925.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc925.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc925.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc925.dist-info/RECORD,,

reconcile/templating/lib/merge_request_manager.py

@@ -136,7 +136,7 @@ class MergeRequestManager(MergeRequestManagerBase[TemplateInfo]):
 
         output = data.data
         collections = {o.input.collection for o in output}
-        collection_hashes = {o.input.collection_hash for o in output}
+        collection_hashes = {o.input.calc_template_hash() for o in output}
         additional_labels = {label for o in output for label in o.input.labels}
         # From the way the code is written, we can assert that there is only one collection and one template hash
         assert len(collections) == 1

reconcile/templating/lib/model.py

@@ -1,12 +1,30 @@
+from typing import Any
+
+from deepdiff import DeepHash
 from pydantic import BaseModel
 
+from reconcile.gql_definitions.templating.template_collection import (
+    TemplateV1,
+)
+
 
 class TemplateInput(BaseModel):
     collection: str
-    collection_hash: str
+    templates: list[TemplateV1] = []
+    variables: list[dict[str, Any]] = []
+    collection_hash: str = ""
     enable_auto_approval: bool = False
     labels: list[str] = []
 
+    def calc_template_hash(self) -> str:
+        if not self.collection_hash:
+            hashable = {
+                "templates": sorted(self.templates, key=lambda x: x.name),
+                "variables": self.variables,
+            }
+            self.collection_hash = DeepHash(hashable)[hashable]
+        return self.collection_hash
+
 
 class TemplateOutput(BaseModel):
     input: TemplateInput

reconcile/templating/renderer.py

@@ -7,7 +7,6 @@ from collections.abc import Callable
 from pathlib import Path
 from typing import Any, Self
 
-from deepdiff import DeepHash
 from ruamel import yaml
 
 from reconcile.gql_definitions.templating.template_collection import (
@@ -187,14 +186,6 @@ def unpack_dynamic_variables(
     return dynamic
 
 
-def calc_template_hash(c: TemplateCollectionV1, variables: dict[str, Any]) -> str:
-    hashable = {
-        "templates": sorted(c.templates, key=lambda x: x.name),
-        "variables": variables,
-    }
-    return DeepHash(hashable)[hashable]
-
-
 class TemplateRendererIntegrationParams(PydanticRunParams):
     clone_repo: bool = False
     app_interface_data_path: str | None
@@ -276,11 +267,11 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         self,
         collection: TemplateCollectionV1,
         gql_api: GqlApi,
-        dry_run: bool,
         persistence: FilePersistence,
         ruamel_instance: yaml.YAML,
+        input: TemplateInput,
         each: dict[str, Any],
-    ) -> None:
+    ) -> list[TemplateOutput]:
         variables = {}
         if collection.variables:
             variables = {
@@ -289,20 +280,17 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
                 ),
                 "static": unpack_static_variables(collection.variables, each),
             }
+            input.variables.append(variables)
 
-        with PersistenceTransaction(persistence, dry_run) as p:
-            input = TemplateInput(
-                collection=collection.name,
-                collection_hash=calc_template_hash(collection, variables),
-                enable_auto_approval=collection.enable_auto_approval or False,
-                labels=collection.additional_mr_labels or [],
+        outputs: list[TemplateOutput] = []
+        for template in collection.templates:
+            output = self.process_template(
+                template, variables, persistence, ruamel_instance, input
             )
-            for template in collection.templates:
-                output = self.process_template(
-                    template, variables, p, ruamel_instance, input
-                )
-                if not dry_run and output:
-                    p.write([output])
+            if output:
+                outputs.append(output)
+
+        return outputs
 
     def reconcile(
         self,
@@ -317,15 +305,27 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
             for_each_items: list[dict[str, Any]] = [{}]
             if c.for_each and c.for_each.items:
                 for_each_items = c.for_each.items
-            for item in for_each_items:
-                self.reconcile_template_collection(
-                    c,
-                    gql_no_validation,
-                    dry_run,
-                    persistence,
-                    ruamel_instance,
-                    item,
-                )
+            input = TemplateInput(
+                collection=c.name,
+                templates=c.templates,
+                enable_auto_approval=c.enable_auto_approval or False,
+                labels=c.additional_mr_labels or [],
+            )
+            with PersistenceTransaction(persistence, dry_run) as p:
+                outputs: list[TemplateOutput] = []
+                for item in for_each_items:
+                    outputs.extend(
+                        self.reconcile_template_collection(
+                            c,
+                            gql_no_validation,
+                            p,
+                            ruamel_instance,
+                            input,
+                            item,
+                        )
+                    )
+                if not dry_run and outputs:
+                    p.write(outputs)
 
     @property
     def name(self) -> str:

reconcile/test/test_saasherder.py

@@ -1085,6 +1085,7 @@ class TestConfigHashPromotionsValidation(TestCase):
             "success": True,
             "saas_file": self.saas_file.name,
             "target_config_hash": "ed2af38cf21f268c",
+            "has_succeeded_once": True,
         }
         self.state_mock.get.return_value = publisher_state
         result = self.saasherder.validate_promotions()
@@ -1101,16 +1102,19 @@ class TestConfigHashPromotionsValidation(TestCase):
                 "success": True,
                 "saas_file": self.saas_file.name,
                 "target_config_hash": "ed2af38cf21f268c",
+                "has_succeeded_once": True,
             },
             {
                 "success": True,
                 "saas_file": self.saas_file.name,
                 "target_config_hash": "ed2af38cf21f268c",
+                "has_succeeded_once": True,
             },
             {
                 "success": True,
                 "saas_file": self.saas_file.name,
                 "target_config_hash": "will_not_match",
+                "has_succeeded_once": True,
             },
         ]
         self.state_mock.get.side_effect = publisher_states
@@ -1137,6 +1141,7 @@ class TestConfigHashPromotionsValidation(TestCase):
             "success": True,
             "saas_file": self.saas_file.name,
             "target_config_hash": "whatever",
+            "has_succeeded_once": True,
         }
 
         self.assertEqual(len(self.saasherder.promotions), 4)
@@ -1148,6 +1153,62 @@ class TestConfigHashPromotionsValidation(TestCase):
         result = self.saasherder.validate_promotions()
         self.assertTrue(result)
 
+    def test_promotion_state_re_deployment_failed(self) -> None:
+        """A promotion is valid if it has ever succeeded for that ref.
+        Re-deployment results should be neglected for validation.
+        """
+        publisher_state = {
+            # Latest state is failed ...
+            "success": False,
+            "saas_file": self.saas_file.name,
+            "target_config_hash": "ed2af38cf21f268c",
+            # ... however, the deployment succeeded sometime before once.
+            "has_succeeded_once": True,
+        }
+        self.state_mock.get.return_value = publisher_state
+        result = self.saasherder.validate_promotions()
+        self.assertTrue(result)
+
+    def test_promotion_state_never_successfully_deployed(self) -> None:
+        """A promotion is invalid, if it never succeeded before."""
+        publisher_state = {
+            # Latest state is failed ...
+            "success": False,
+            "saas_file": self.saas_file.name,
+            "target_config_hash": "ed2af38cf21f268c",
+            # ... and it never succeeded once before.
+            "has_succeeded_once": False,
+        }
+        self.state_mock.get.return_value = publisher_state
+        result = self.saasherder.validate_promotions()
+        self.assertFalse(result)
+
+    def test_promotion_state_success_backwards_compatibility_success(self) -> None:
+        """Not all states have the has_succeeded_once attribute yet.
+        If it doesnt exist, we should always fall back to latest success state.
+        """
+        publisher_state = {
+            "success": True,
+            "saas_file": self.saas_file.name,
+            "target_config_hash": "ed2af38cf21f268c",
+        }
+        self.state_mock.get.return_value = publisher_state
+        result = self.saasherder.validate_promotions()
+        self.assertTrue(result)
+
+    def test_promotion_state_success_backwards_compatibility_fail(self) -> None:
+        """Not all states have the has_succeeded_once attribute yet.
+        If it doesnt exist, we should always fall back to latest success state.
+        """
+        publisher_state = {
+            "success": False,
+            "saas_file": self.saas_file.name,
+            "target_config_hash": "ed2af38cf21f268c",
+        }
+        self.state_mock.get.return_value = publisher_state
+        result = self.saasherder.validate_promotions()
+        self.assertFalse(result)
+
 
 @pytest.mark.usefixtures("inject_gql_class_factory")
 class TestSoakDays(TestCase):

reconcile/utils/promotion_state.py

@@ -18,10 +18,15 @@ class PromotionData(BaseModel):
     requirements.
     """
 
+    # The success is primarily used for SAPM auto-promotions
     success: bool
     target_config_hash: str | None
     saas_file: str | None
     check_in: str | None
+    # Whether this promotion has ever succeeded
+    # Note, this shouldnt be overridden on subsequent promotions of same ref
+    # This attribute is primarily used by saasherder validations
+    has_succeeded_once: bool | None
 
     class Config:
         smart_union = True

reconcile/utils/saasherder/saasherder.py

@@ -1846,7 +1846,9 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     target_uid=target_uid,
                     pre_check_sha_exists=False,
                 )
-                if not (deployment and deployment.success):
+                if not (
+                    deployment and (deployment.success or deployment.has_succeeded_once)
+                ):
                     logging.error(
                         f"Commit {promotion.commit_sha} was not "
                         + f"published with success to channel {channel.name}"
@@ -1943,6 +1945,17 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                 all_subscribed_saas_file_paths = set()
                 all_subscribed_target_paths = set()
                 for channel in promotion.publish:
+                    # make sure we keep some attributes on re-deployments of same ref
+                    has_succeeded_once = success
+                    current_state = self._promotion_state.get_promotion_data(
+                        sha=promotion.commit_sha,
+                        channel=channel,
+                        target_uid=promotion.saas_target_uid,
+                        use_cache=True,
+                    )
+                    if current_state and current_state.has_succeeded_once:
+                        has_succeeded_once = True
+
                     # publish to state to pass promotion gate
                     self._promotion_state.publish_promotion_data(
                         sha=promotion.commit_sha,
@@ -1952,6 +1965,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                             saas_file=promotion.saas_file,
                             success=success,
                             target_config_hash=promotion.target_config_hash,
+                            has_succeeded_once=has_succeeded_once,
                             # TODO: do not override - check if timestamp already exists
                             check_in=str(now),
                         ),