qontract-reconcile 0.10.1rc514__py3-none-any.whl → 0.10.1rc515__py3-none-any.whl

{qontract_reconcile-0.10.1rc514.dist-info → qontract_reconcile-0.10.1rc515.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc514
+Version: 0.10.1rc515
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc514.dist-info → qontract_reconcile-0.10.1rc515.dist-info}/RECORD

@@ -345,14 +345,16 @@ reconcile/rhidp/sso_client/base.py,sha256=EfQ2ewcOKh5idg46UKAkY6z0m_nGQfvnQKffa2
 reconcile/rhidp/sso_client/integration.py,sha256=kA8g7c38ZBSdrRtyfEqy_WgSreD1PbwY7ZIN-3tZRPc,2221
 reconcile/rhidp/sso_client/metrics.py,sha256=Tq7tSOsqL3XdcPUdozxqzSPIodUeOV87UCTqpuuqqhw,1013
 reconcile/saas_auto_promotions_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/saas_auto_promotions_manager/integration.py,sha256=5hSvC3JUM4r63UFuC1IbeG6YbmdlzQrekwRqF5ggvCY,5667
+reconcile/saas_auto_promotions_manager/integration.py,sha256=1nm3aV5a0frEPTSww4ue1mfQ_Px6XaRZoaBRhPgJiAU,6472
 reconcile/saas_auto_promotions_manager/publisher.py,sha256=4_M9Oykhj-kEZPUn05E2DY5gD6-x32Dgf7K3NPOkGEg,2029
 reconcile/saas_auto_promotions_manager/subscriber.py,sha256=cLhPlkT71J2LIice3SLmH1WpsqzV46gd0peMxrnqyRw,7452
 reconcile/saas_auto_promotions_manager/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request.py,sha256=PNu7sE-tDUY61E03z5w0b93fdowZ8auCl0S4_vhYOKQ,1333
-reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager.py,sha256=CpTbL2wX4s6iQVZJzu-i8kTJOezCwPxyJTMoNdltqgc,7525
-reconcile/saas_auto_promotions_manager/merge_request_manager/mr_parser.py,sha256=gyEn7yp03XDf0P8QEDfXWTcGGehNQ2Ln_sQ3cUq00Bc,6945
-reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py,sha256=Sj3mwmBcml6t-49bPnXdq7snT9N0WKrPk2chU_0ylyE,7188
+reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager.py,sha256=P4RH6y-qC9RKJxVguyOfOAe42a6AP1Ez-NlyMSH3j8I,7492
+reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager_v2.py,sha256=AwCyvnNAKwRbESodHN2-fWG8Wpxua0nzQkcIsz4Gevg,5411
+reconcile/saas_auto_promotions_manager/merge_request_manager/mr_parser.py,sha256=x8Gg-YjEFWEeDPJH3Y8SrfcJbwhLuAqCz4kIhfEyaaA,7060
+reconcile/saas_auto_promotions_manager/merge_request_manager/reconciler.py,sha256=DgMw24exjuor9s4Y8y0Ftsjs5l7rl8D7dksMoJ1RF0A,7639
+reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py,sha256=VSKCwY8jZ6NYwIT6edmVnpzY7cgjhn5fhX9xtlw2LD8,7341
 reconcile/saas_auto_promotions_manager/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/saas_auto_promotions_manager/utils/saas_files_inventory.py,sha256=mihuWynroB1Cea1Lsvf6V8Nb8PGiBdcLC0uhCX_52Y0,6966
 reconcile/skupper_network/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -452,14 +454,14 @@ reconcile/test/test_vpc_peerings_validator.py,sha256=dFSmjc_dMN2GqMbntCFpa7PUZmy
 reconcile/test/test_wrong_region.py,sha256=7KzL7OaICQ9Z3DW27zt_ykMN7_87owAFC-2CYjvGoyA,2138
 reconcile/test/saas_auto_promotions_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/saas_auto_promotions_manager/conftest.py,sha256=tZNs35EuWulP53Cqt61RteOGY_uH4gfhtXfGQ8-awwM,3081
-reconcile/test/saas_auto_promotions_manager/test_integration_test.py,sha256=4LPWN7YWr2t_MRpwGf7pQ4AMSBEMYvTqTe5hY1rJyS0,4078
+reconcile/test/saas_auto_promotions_manager/test_integration_test.py,sha256=UIafVpeof9Zd3Q17mEDIUwgdBGf8oFxkf7h8tLfmwCE,2077
 reconcile/test/saas_auto_promotions_manager/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/conftest.py,sha256=Ky5IAi0EzrHqP0eimV4Mq0FMVdrFEN-7P-Nst4wqB34,4276
-reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/data_keys.py,sha256=ZTGXlpgAECZnNqsJbkYGr6fBreC_KejYz4tCJIg8iDM,573
-reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_merge_request_manager.py,sha256=c0-c9wfd-3mOMFsCJUDOCx-EfoiT75hJypz72xwdMb8,5978
-reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_mr_parser.py,sha256=9VLJKOW80NMQ3WCKDqtaOa0-StoFxaCzKN-NCPfeklo,9845
-reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_unbatching.py,sha256=IqmVqWw98Gegwz5johzh13cqCONk7D_VbgK1y8VqRP4,3004
+reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/conftest.py,sha256=7O3lbk1EmEtUofqGncfiwMYvDPXrkQNPB59zlQ_zXkM,4588
+reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/data_keys.py,sha256=Z1IV51OUuzhd-3S8W-k7ixC-fkaglCokn0eakK0Z73s,606
+reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_merge_request_manager.py,sha256=ryROiiQNtZvuG820CB-RQ9FMmLAGshezyMDNDiJNA_E,2369
+reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_mr_parser.py,sha256=dcGHzxuafKSxmswSO1qF2WlKaqsmEvtERC6Lb8kDAN0,10019
+reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_reconciler.py,sha256=-rSv-C-JFaY0gsjSFgcXz9ASNQQOzwizaQEbVYJtLHg,14906
 reconcile/test/saas_auto_promotions_manager/merge_request_manager/renderer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/saas_auto_promotions_manager/merge_request_manager/renderer/conftest.py,sha256=2rCSstewp4LPoEJHm5N7dGJexEtY8ndLHvoGZYjmpsc,1678
 reconcile/test/saas_auto_promotions_manager/merge_request_manager/renderer/data_keys.py,sha256=beHYQ9kgDLeBZgC2FvxQA3tHx1PO-RAMN8_kVcSdikI,90
@@ -666,8 +668,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=d18KrdhtUGqoC7_kWZU128U0-VJEj-0rjFkLVufcI6I,2755
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc514.dist-info/METADATA,sha256=-Ior_MrT3HmyZ-QKjTJqbS8J30opXi5LPdBud6CU0dg,2349
-qontract_reconcile-0.10.1rc514.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qontract_reconcile-0.10.1rc514.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
-qontract_reconcile-0.10.1rc514.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc514.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc515.dist-info/METADATA,sha256=Bgwt_oRZlcfCQ6ycYlopR_xt-fRVu3kWrlOW7wMuaqo,2349
+qontract_reconcile-0.10.1rc515.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qontract_reconcile-0.10.1rc515.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
+qontract_reconcile-0.10.1rc515.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc515.dist-info/RECORD,,

reconcile/saas_auto_promotions_manager/integration.py

@@ -9,9 +9,15 @@ from reconcile.openshift_saas_deploy import (
 from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager import (
     MergeRequestManager,
 )
+from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager_v2 import (
+    MergeRequestManagerV2,
+)
 from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
     MRParser,
 )
+from reconcile.saas_auto_promotions_manager.merge_request_manager.reconciler import (
+    Reconciler,
+)
 from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
     Renderer,
 )
@@ -45,6 +51,7 @@ class SaasAutoPromotionsManager:
         deployment_state: PromotionState,
         vcs: VCS,
         saas_file_inventory: SaasFilesInventory,
+        merge_request_manager_v2: MergeRequestManagerV2,
         merge_request_manager: MergeRequestManager,
         thread_pool_size: int,
         dry_run: bool,
@@ -52,6 +59,7 @@ class SaasAutoPromotionsManager:
         self._deployment_state = deployment_state
         self._vcs = vcs
         self._saas_file_inventory = saas_file_inventory
+        self._merge_request_manager_v2 = merge_request_manager_v2
         self._merge_request_manager = merge_request_manager
         self._thread_pool_size = thread_pool_size
         self._dry_run = dry_run
@@ -88,11 +96,14 @@ class SaasAutoPromotionsManager:
         self._merge_request_manager.create_promotion_merge_requests(
             subscribers=subscribers_with_diff
         )
+        self._merge_request_manager_v2.reconcile(subscribers=subscribers_with_diff)
 
 
 def init_external_dependencies(
     dry_run: bool,
-) -> tuple[PromotionState, VCS, SaasFilesInventory, MergeRequestManager]:
+) -> tuple[
+    PromotionState, VCS, SaasFilesInventory, MergeRequestManagerV2, MergeRequestManager
+]:
     """
     Lets initialize everything that involves calls to external dependencies:
     - VCS -> Gitlab / Github queries
@@ -120,17 +131,29 @@ def init_external_dependencies(
         allow_opening_mrs=allow_opening_mrs,
     )
     mr_parser = MRParser(vcs=vcs)
-    merge_request_manager = MergeRequestManager(
+    merge_request_manager_v2 = MergeRequestManagerV2(
         vcs=vcs,
+        reconciler=Reconciler(),
         mr_parser=mr_parser,
         renderer=Renderer(),
     )
+    merge_request_manager = MergeRequestManager(
+        mr_parser=mr_parser,
+        renderer=Renderer(),
+        vcs=vcs,
+    )
     saas_files = get_saas_files()
     saas_inventory = SaasFilesInventory(saas_files=saas_files)
     deployment_state = PromotionState(
         state=init_state(integration=OPENSHIFT_SAAS_DEPLOY, secret_reader=secret_reader)
     )
-    return deployment_state, vcs, saas_inventory, merge_request_manager
+    return (
+        deployment_state,
+        vcs,
+        saas_inventory,
+        merge_request_manager_v2,
+        merge_request_manager,
+    )
 
 
 @defer
@@ -143,6 +166,7 @@ def run(
         deployment_state,
         vcs,
         saas_inventory,
+        merge_request_manager_v2,
         merge_request_manager,
     ) = init_external_dependencies(dry_run=dry_run)
     if defer:
@@ -153,6 +177,7 @@ def run(
         vcs=vcs,
         saas_file_inventory=saas_inventory,
         merge_request_manager=merge_request_manager,
+        merge_request_manager_v2=merge_request_manager_v2,
         thread_pool_size=thread_pool_size,
         dry_run=dry_run,
     )

reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager.py

@@ -16,7 +16,6 @@ from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer impor
     CHANNELS_REF,
     CONTENT_HASHES,
     IS_BATCHABLE,
-    SAPM_LABEL,
     VERSION_REF,
     Renderer,
 )
@@ -25,6 +24,8 @@ from reconcile.utils.vcs import VCS
 
 ITEM_SEPARATOR = ","
 
+SAPM_LABEL = "SAPM"
+
 
 class MergeRequestManager:
     """
@@ -68,14 +69,13 @@ class MergeRequestManager:
                     "Closing this MR because it failed MR check and isn't marked un-batchable yet.",
                 )
                 # Remember these hashes as unbatchable
-                content_hashes = mr.content_hashes.split(ITEM_SEPARATOR)
-                self._unbatchable_hashes.update(content_hashes)
+                self._unbatchable_hashes.update(mr.content_hashes)
             else:
                 open_mrs_after_unbatching.append(mr)
         self._open_mrs = open_mrs_after_unbatching
 
     def housekeeping(self) -> None:
-        self._open_mrs = self._mr_parser.retrieve_open_mrs()
+        self._open_mrs = self._mr_parser.retrieve_open_mrs(label=SAPM_LABEL)
         self._unbatch_failed_mrs()
 
     def _aggregate_subscribers_per_channel_combo(
@@ -160,7 +160,7 @@ class MergeRequestManager:
                 channels=channel_combo,
                 is_batchable=combined_content_hash not in self._unbatchable_hashes,
             )
-            title = self._renderer.render_title(channels=channel_combo)
+            title = self._renderer.render_title(is_draft=False, channels=channel_combo)
             logging.info(
                 "Open MR for update in channel(s) %s",
                 channel_combo,

reconcile/saas_auto_promotions_manager/merge_request_manager/merge_request_manager_v2.py

@@ -0,0 +1,144 @@
+import logging
+from collections import defaultdict
+from collections.abc import Iterable
+
+from gitlab.exceptions import GitlabGetError
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request import (
+    SAPMMR,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
+    MRParser,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.reconciler import (
+    Addition,
+    Promotion,
+    Reconciler,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
+    Renderer,
+)
+from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
+from reconcile.utils.vcs import VCS
+
+BATCH_SIZE_LIMIT = 5
+SAPM_LABEL = "SAPMCanary"
+
+
+class MergeRequestManagerV2:
+    """
+    Manager for SAPM merge requests.
+
+    This class uses MRParser to fetch current state (currently open MRs).
+    This class calculates the desired state (i.e., desired promotions).
+    Desired state and current state are given to the Reconciler, which will
+    then determine a Diff (Additions and Deletions of MRs).
+
+    This class interacts with VCS to realize the result of the Diff.
+    """
+
+    def __init__(
+        self, vcs: VCS, mr_parser: MRParser, reconciler: Reconciler, renderer: Renderer
+    ):
+        self._vcs = vcs
+        self._mr_parser = mr_parser
+        self._renderer = renderer
+        self._reconciler = reconciler
+        self._content_hash_to_subscriber: dict[str, list[Subscriber]] = {}
+        self._sapm_mrs: list[SAPMMR] = []
+
+    def _aggregate_desired_state(
+        self, subscribers: Iterable[Subscriber]
+    ) -> list[Promotion]:
+        subscribers_per_channel_combo: dict[str, list[Subscriber]] = defaultdict(list)
+        for subscriber in subscribers:
+            channel_combo = ",".join([c.name for c in subscriber.channels])
+            subscribers_per_channel_combo[channel_combo].append(subscriber)
+
+        desired_promotions: list[Promotion] = []
+        for channel_combo, subs in subscribers_per_channel_combo.items():
+            combined_content_hash = Subscriber.combined_content_hash(subscribers=subs)
+            self._content_hash_to_subscriber[combined_content_hash] = subs
+            desired_promotions.append(
+                Promotion(
+                    content_hashes={combined_content_hash},
+                    channels={channel_combo},
+                )
+            )
+        return desired_promotions
+
+    def _render_mr(self, addition: Addition) -> None:
+        subs: list[Subscriber] = []
+        for content_hash in addition.content_hashes:
+            subs.extend(self._content_hash_to_subscriber[content_hash])
+        content_by_path: dict[str, str] = {}
+        has_error = False
+        for sub in subs:
+            if sub.target_file_path not in content_by_path:
+                try:
+                    content_by_path[sub.target_file_path] = (
+                        self._vcs.get_file_content_from_app_interface_master(
+                            file_path=sub.target_file_path
+                        )
+                    )
+                except GitlabGetError as e:
+                    if e.response_code == 404:
+                        logging.error(
+                            "The saas file %s does not exist anylonger. Most likely qontract-server data not in synch. This should resolve soon on its own.",
+                            sub.target_file_path,
+                        )
+                        has_error = True
+                        break
+                    raise e
+            content_by_path[sub.target_file_path] = (
+                self._renderer.render_merge_request_content(
+                    subscriber=sub,
+                    current_content=content_by_path[sub.target_file_path],
+                )
+            )
+        if has_error:
+            return
+
+        description_hashes = ",".join(addition.content_hashes)
+        description_channels = ",".join(addition.channels)
+
+        description = self._renderer.render_description(
+            content_hashes=description_hashes,
+            channels=description_channels,
+            is_batchable=addition.batchable,
+        )
+        title = self._renderer.render_title(
+            is_draft=True, channels=description_channels
+        )
+        logging.info(
+            "Open MR for update in channel(s) %s",
+            description_channels,
+        )
+        self._sapm_mrs.append(
+            SAPMMR(
+                sapm_label=SAPM_LABEL,
+                content_by_path=content_by_path,
+                title=title,
+                description=description,
+            )
+        )
+
+    def reconcile(self, subscribers: Iterable[Subscriber]) -> None:
+        current_state = self._mr_parser.retrieve_open_mrs(label=SAPM_LABEL)
+        desired_state = self._aggregate_desired_state(subscribers=subscribers)
+        diff = self._reconciler.reconcile(
+            batch_limit=BATCH_SIZE_LIMIT,
+            desired_promotions=desired_state,
+            open_mrs=current_state,
+        )
+        for deletion in diff.deletions:
+            self._vcs.close_app_interface_mr(
+                mr=deletion.mr.raw,
+                comment=deletion.reason,
+            )
+
+        for addition in diff.additions:
+            self._render_mr(addition=addition)
+
+        for rendered_mr in self._sapm_mrs:
+            self._vcs.open_app_interface_merge_request(mr=rendered_mr)

reconcile/saas_auto_promotions_manager/merge_request_manager/mr_parser.py

@@ -9,7 +9,6 @@ from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer impor
     CONTENT_HASHES,
     IS_BATCHABLE,
     PROMOTION_DATA_SEPARATOR,
-    SAPM_LABEL,
     SAPM_VERSION,
     VERSION_REF,
 )
@@ -19,12 +18,15 @@ from reconcile.utils.vcs import VCS, MRCheckStatus
 @dataclass
 class OpenMergeRequest:
     raw: ProjectMergeRequest
-    content_hashes: str
-    channels: str
+    content_hashes: set[str]
+    channels: set[str]
     failed_mr_check: bool
     is_batchable: bool
 
 
+ITEM_SEPARATOR = ","
+
+
 class MRParser:
     """
     We store state in MR descriptions.
@@ -49,11 +51,13 @@ class MRParser:
             return ""
         return groups[0]
 
-    def _fetch_sapm_managed_open_merge_requests(self) -> list[ProjectMergeRequest]:
+    def _fetch_sapm_managed_open_merge_requests(
+        self, label: str
+    ) -> list[ProjectMergeRequest]:
         all_open_mrs = self._vcs.get_open_app_interface_merge_requests()
-        return [mr for mr in all_open_mrs if SAPM_LABEL in mr.attributes.get("labels")]
+        return [mr for mr in all_open_mrs if label in mr.attributes.get("labels")]
 
-    def retrieve_open_mrs(self) -> list[OpenMergeRequest]:
+    def retrieve_open_mrs(self, label: str) -> list[OpenMergeRequest]:
         """
         This function parses the state and returns a list of valid, parsed open MRs (current state).
         If any issue is encountered during parsing, we consider this MR
@@ -66,7 +70,7 @@ class MRParser:
         """
         open_mrs: list[OpenMergeRequest] = []
         seen: set[tuple[str, str, str]] = set()
-        for mr in self._fetch_sapm_managed_open_merge_requests():
+        for mr in self._fetch_sapm_managed_open_merge_requests(label=label):
             attrs = mr.attributes
             desc = attrs.get("description")
             has_conflicts = attrs.get("has_conflicts", False)
@@ -177,8 +181,8 @@ class MRParser:
             open_mrs.append(
                 OpenMergeRequest(
                     raw=mr,
-                    content_hashes=content_hashes,
-                    channels=channels_refs,
+                    content_hashes=set(content_hashes.split(ITEM_SEPARATOR)),
+                    channels=set(channels_refs.split(ITEM_SEPARATOR)),
                     failed_mr_check=mr_check_status == MRCheckStatus.FAILED,
                     is_batchable=is_batchable_str == "True",
                 )

reconcile/saas_auto_promotions_manager/merge_request_manager/reconciler.py

@@ -0,0 +1,207 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Optional
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
+    OpenMergeRequest,
+)
+
+MSG_MISSING_UNBATCHING = (
+    "Closing this MR because it failed MR check and isn't marked as un-batchable yet."
+)
+MSG_OUTDATED_CONTENT = "Closing this MR because it has out-dated content."
+MSG_NEW_BATCH = "Closing this MR in favor of a new batch MR."
+
+
+@dataclass
+class Promotion:
+    content_hashes: set[str]
+    channels: set[str]
+
+
+@dataclass
+class Deletion:
+    mr: OpenMergeRequest
+    reason: str
+
+
+@dataclass
+class Addition:
+    content_hashes: set[str]
+    channels: set[str]
+    batchable: bool
+
+
+@dataclass
+class Diff:
+    deletions: list[Deletion]
+    additions: list[Addition]
+
+
+class Reconciler:
+    """
+    The reconciler calculates a Diff. I.e., which MRs need to be opened (Addition)
+    and which MRs need to be closed (Deletion). The reconciler has no external
+    dependencies and does not interact with VCS. The reconciler expects to be
+    given the desired state (which promotions do we want) and the current state
+    (the currently open MRs) in order to calculate the Diff.
+    """
+
+    def __init__(self) -> None:
+        self._desired_promotions: Iterable[Promotion] = []
+        self._open_mrs: Iterable[OpenMergeRequest] = []
+
+    def _unbatch(self, diff: Diff) -> None:
+        """
+        We optimistically batch MRs together that didnt run through MR check yet.
+        Vast majority of auto-promotion MRs are succeeding checks, so we can stay optimistic.
+        In the rare case of an MR failing the check, we want to unbatch it.
+        I.e., we open a dedicated MR for each channel in the batched MR, mark the new MRs as non-batchable
+        and close the old batched MR. By doing so, we ensure that unrelated MRs are not blocking each other.
+        Unbatched MRs are marked and will never be batched again.
+        """
+        open_mrs_after_unbatching: list[OpenMergeRequest] = []
+        unbatchable_hashes: set[str] = set()
+        falsely_marked_batchable_hashes: set[str] = set()
+        for mr in self._open_mrs:
+            if not mr.is_batchable:
+                unbatchable_hashes.update(mr.content_hashes)
+                open_mrs_after_unbatching.append(mr)
+            elif mr.failed_mr_check:
+                falsely_marked_batchable_hashes.update(mr.content_hashes)
+                diff.deletions.append(
+                    Deletion(
+                        mr=mr,
+                        reason=MSG_MISSING_UNBATCHING,
+                    )
+                )
+            else:
+                open_mrs_after_unbatching.append(mr)
+        self._open_mrs = open_mrs_after_unbatching
+
+        desired_promotions_after_unbatching: list[Promotion] = []
+        for promotion in self._desired_promotions:
+            if promotion.content_hashes.issubset(unbatchable_hashes):
+                continue
+            elif promotion.content_hashes.issubset(falsely_marked_batchable_hashes):
+                diff.additions.append(
+                    Addition(
+                        content_hashes=promotion.content_hashes,
+                        channels=promotion.channels,
+                        batchable=False,
+                    )
+                )
+                continue
+            else:
+                desired_promotions_after_unbatching.append(promotion)
+        self._desired_promotions = desired_promotions_after_unbatching
+
+    def _remove_outdated(self, diff: Diff) -> None:
+        """
+        We want to be sure that the open MRs are still addressing desired content.
+        We close MRs that are not addressing any content hash in a desired promotion.
+        """
+        all_desired_content_hashes: set[str] = set()
+        for promotion in self._desired_promotions:
+            all_desired_content_hashes.update(promotion.content_hashes)
+
+        open_mrs_after_deletion: list[OpenMergeRequest] = []
+        for mr in self._open_mrs:
+            if mr.content_hashes.issubset(all_desired_content_hashes):
+                open_mrs_after_deletion.append(mr)
+                continue
+            diff.deletions.append(
+                Deletion(
+                    mr=mr,
+                    reason=MSG_OUTDATED_CONTENT,
+                )
+            )
+        self._open_mrs = open_mrs_after_deletion
+
+    def _batch_remaining_mrs(self, diff: Diff, batch_limit: int) -> None:
+        """
+        Remaining desired promotions should be batched together
+        if they are not addressed yet by a valid open MR.
+
+        We do not want to let MR batches grow infinitely,
+        as constant change of a batch might starve MRs that
+        are already part of the batch.
+        In order for MRs to not grow infinitely, we apply a
+        BATCH_LIMIT per MR, i.e., a maximum number of hashes
+        that can be batched together.
+        """
+        submitted_content_hashes: set[str] = set()
+        for open_mr in self._open_mrs:
+            submitted_content_hashes.update(open_mr.content_hashes)
+
+        unsubmitted_promotions = [
+            prom
+            for prom in self._desired_promotions
+            if not prom.content_hashes.issubset(submitted_content_hashes)
+        ]
+
+        if not unsubmitted_promotions:
+            return
+
+        batch_with_capacity: Optional[OpenMergeRequest] = None
+        for mr in self._open_mrs:
+            if len(mr.content_hashes) < batch_limit:
+                batch_with_capacity = mr
+                # Note, there should always only be maximum one batch with capacity available
+                break
+
+        if batch_with_capacity:
+            # We disassemble the batch to its promotions
+            # can be added to new batch(es)
+            unsubmitted_promotions.append(
+                Promotion(
+                    content_hashes=batch_with_capacity.content_hashes,
+                    channels=batch_with_capacity.channels,
+                )
+            )
+            # Lets close the current batch so remaining promotions can
+            # be aggregated in new batch(es)
+            diff.deletions.append(
+                Deletion(
+                    mr=batch_with_capacity,
+                    reason=MSG_NEW_BATCH,
+                )
+            )
+
+        batched_mr = Addition(
+            content_hashes=set(),
+            channels=set(),
+            batchable=True,
+        )
+
+        for promotion in unsubmitted_promotions:
+            batched_mr.content_hashes.update(promotion.content_hashes)
+            batched_mr.channels.update(promotion.channels)
+            if len(batched_mr.content_hashes) >= batch_limit:
+                # Note, we might also be above the batch limit, but thats ok.
+                # We only ensure that we create a new batch now and dont grow further.
+                diff.additions.append(batched_mr)
+                batched_mr = Addition(
+                    content_hashes=set(),
+                    channels=set(),
+                    batchable=True,
+                )
+        if batched_mr.content_hashes:
+            diff.additions.append(batched_mr)
+
+    def reconcile(
+        self,
+        desired_promotions: Iterable[Promotion],
+        open_mrs: Iterable[OpenMergeRequest],
+        batch_limit: int,
+    ) -> Diff:
+        self._open_mrs = open_mrs
+        self._desired_promotions = desired_promotions
+        diff = Diff(
+            deletions=[],
+            additions=[],
+        )
+        self._unbatch(diff=diff)
+        self._remove_outdated(diff=diff)
+        self._batch_remaining_mrs(diff=diff, batch_limit=batch_limit)
+        return diff

reconcile/saas_auto_promotions_manager/merge_request_manager/renderer.py

@@ -1,3 +1,4 @@
+import hashlib
 from collections.abc import Mapping
 from copy import deepcopy
 from typing import Any
@@ -18,18 +19,17 @@ from reconcile.saas_auto_promotions_manager.subscriber import Subscriber
 PROMOTION_DATA_SEPARATOR = (
     "**SAPM Data - DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
 )
-SAPM_VERSION = "1.1.0"
-SAPM_LABEL = "SAPM"
+SAPM_VERSION = "2.0.0"
 CONTENT_HASHES = "content_hashes"
 CHANNELS_REF = "channels"
 IS_BATCHABLE = "is_batchable"
 VERSION_REF = "sapm_version"
-SAPM_DESC = f"""
+SAPM_DESC = """
 This is an auto-promotion triggered by app-interface's [saas-auto-promotions-manager](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/saas_auto_promotions_manager) (SAPM).
 The channel(s) mentioned in the MR title had an event.
 This MR promotes all subscribers with auto-promotions for these channel(s).
 
-Please **do not remove the {SAPM_LABEL} label** from this MR.
+Please **do not remove or change any label** from this MR.
 
 Parts of this description are used by SAPM to manage auto-promotions.
 """
@@ -152,8 +152,11 @@ class Renderer:
 {VERSION_REF}: {SAPM_VERSION}
         """
 
-    def render_title(self, channels: str) -> str:
-        return f"[SAPM] auto-promotion for {channels}"
+    def render_title(self, is_draft: bool, channels: str) -> str:
+        content = f"[SAPM] auto-promotion ID {int(hashlib.sha256(channels.encode('utf-8')).hexdigest(), 16) % 10**8}"
+        if is_draft:
+            return f"Draft: {content}"
+        return content
 
 
 def _parse_expression(expression: str) -> Any: