qontract-reconcile 0.10.1rc513__py3-none-any.whl → 0.10.1rc515__py3-none-any.whl

reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_reconciler.py

@@ -0,0 +1,408 @@
+from collections.abc import Sequence
+from unittest.mock import create_autospec
+
+import pytest
+from gitlab.v4.objects import ProjectMergeRequest
+
+from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
+    OpenMergeRequest,
+)
+from reconcile.saas_auto_promotions_manager.merge_request_manager.reconciler import (
+    MSG_MISSING_UNBATCHING,
+    MSG_NEW_BATCH,
+    MSG_OUTDATED_CONTENT,
+    Addition,
+    Deletion,
+    Diff,
+    Promotion,
+    Reconciler,
+)
+
+
+def _aggregate_hashes(items: Sequence[Addition | Deletion]) -> set[str]:
+    hashes: set[str] = set()
+    for item in items:
+        if isinstance(item, Addition):
+            hashes.update(item.content_hashes)
+        else:
+            hashes.update(item.mr.content_hashes)
+    return hashes
+
+
+def _aggregate_channels(items: Sequence[Addition | Deletion]) -> set[str]:
+    channels: set[str] = set()
+    for item in items:
+        if isinstance(item, Addition):
+            channels.update(item.channels)
+        else:
+            channels.update(item.mr.channels)
+    return channels
+
+
+@pytest.mark.parametrize(
+    "desired_promotions, open_mrs, expected_diff",
+    [
+        # No open MRs. Aggregate desired promotions into single batched Promotion
+        (
+            [
+                Promotion(
+                    channels={"chan1"},
+                    content_hashes={"hash1"},
+                ),
+                Promotion(
+                    channels={"chan2", "chan3"},
+                    content_hashes={"hash2", "hash3"},
+                ),
+            ],
+            [],
+            Diff(
+                deletions=[],
+                additions=[
+                    Addition(
+                        channels={"chan1", "chan2", "chan3"},
+                        content_hashes={"hash1", "hash2", "hash3"},
+                        batchable=True,
+                    ),
+                ],
+            ),
+        ),
+        # No desired promotions. We expect that every MR gets closed,
+        # no matter its current state.
+        (
+            [],
+            [
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan1", "chan2"},
+                    content_hashes={"hash1", "hash2"},
+                    failed_mr_check=True,
+                    is_batchable=True,
+                ),
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan3"},
+                    content_hashes={"hash3"},
+                    failed_mr_check=False,
+                    is_batchable=True,
+                ),
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan4"},
+                    content_hashes={"hash4"},
+                    failed_mr_check=False,
+                    is_batchable=False,
+                ),
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan5"},
+                    content_hashes={"hash5"},
+                    failed_mr_check=True,
+                    is_batchable=False,
+                ),
+            ],
+            Diff(
+                deletions=[
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan1", "chan2"},
+                            content_hashes={"hash1", "hash2"},
+                            failed_mr_check=True,
+                            is_batchable=True,
+                        ),
+                        reason=MSG_MISSING_UNBATCHING,
+                    ),
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan3"},
+                            content_hashes={"hash3"},
+                            failed_mr_check=False,
+                            is_batchable=True,
+                        ),
+                        reason=MSG_OUTDATED_CONTENT,
+                    ),
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan4"},
+                            content_hashes={"hash4"},
+                            failed_mr_check=False,
+                            is_batchable=False,
+                        ),
+                        reason=MSG_OUTDATED_CONTENT,
+                    ),
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan5"},
+                            content_hashes={"hash5"},
+                            failed_mr_check=True,
+                            is_batchable=False,
+                        ),
+                        reason=MSG_OUTDATED_CONTENT,
+                    ),
+                ],
+                additions=[],
+            ),
+        ),
+        # We have a single failed, but still marked as batchable MR.
+        # The hashes in the failed MR are still desired state.
+        # We expect this MR to be closed. Further, a new MR should be
+        # opened that is marked as unbatchable. Each hash should have
+        # its own separated MR.
+        (
+            [
+                Promotion(
+                    channels={"chan1"},
+                    content_hashes={"hash1"},
+                ),
+                Promotion(
+                    channels={"chan2"},
+                    content_hashes={"hash2"},
+                ),
+            ],
+            [
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan1", "chan2"},
+                    content_hashes={"hash1", "hash2"},
+                    failed_mr_check=True,
+                    is_batchable=True,
+                ),
+            ],
+            Diff(
+                deletions=[
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan1", "chan2"},
+                            content_hashes={"hash1", "hash2"},
+                            failed_mr_check=True,
+                            is_batchable=True,
+                        ),
+                        reason=MSG_MISSING_UNBATCHING,
+                    )
+                ],
+                additions=[
+                    Addition(
+                        channels={"chan1"},
+                        content_hashes={"hash1"},
+                        batchable=False,
+                    ),
+                    Addition(
+                        channels={"chan2"},
+                        content_hashes={"hash2"},
+                        batchable=False,
+                    ),
+                ],
+            ),
+        ),
+        # We have an open valid batched MR. All desired promotions
+        # are already addressed by that MR -> there is nothing to do.
+        (
+            [
+                Promotion(
+                    channels={"chan1"},
+                    content_hashes={"hash1"},
+                ),
+                Promotion(
+                    channels={"chan2"},
+                    content_hashes={"hash2"},
+                ),
+            ],
+            [
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan1", "chan2"},
+                    content_hashes={"hash1", "hash2"},
+                    failed_mr_check=False,
+                    is_batchable=True,
+                ),
+            ],
+            Diff(
+                deletions=[],
+                additions=[],
+            ),
+        ),
+        # We have an open valid batched MR. However, there is a promotion
+        # that is not addressed by the open MR.
+        # We expect that the MR gets closed and a new
+        # batched MR gets created which addresses all desired promotions.
+        (
+            [
+                Promotion(
+                    channels={"chan1"},
+                    content_hashes={"hash1"},
+                ),
+                Promotion(
+                    channels={"chan2"},
+                    content_hashes={"hash2"},
+                ),
+                Promotion(
+                    channels={"chan3", "chan4"},
+                    content_hashes={"hash3", "hash4"},
+                ),
+            ],
+            [
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan1", "chan2"},
+                    content_hashes={"hash1", "hash2"},
+                    failed_mr_check=False,
+                    is_batchable=True,
+                ),
+            ],
+            Diff(
+                deletions=[
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan1", "chan2"},
+                            content_hashes={"hash1", "hash2"},
+                            failed_mr_check=False,
+                            is_batchable=True,
+                        ),
+                        reason=MSG_NEW_BATCH,
+                    )
+                ],
+                additions=[
+                    Addition(
+                        content_hashes={"hash1", "hash2", "hash3", "hash4"},
+                        channels={"chan1", "chan2", "chan3", "chan4"},
+                        batchable=True,
+                    )
+                ],
+            ),
+        ),
+        # We want to promote 12 content hashes.
+        # We surpass the current batch limit of 5.
+        # We expect a total of 3 batch MRs to be created.
+        # The existing open batch MR should be closed.
+        (
+            [
+                Promotion(
+                    channels={"chan1", "chan2", "chan3"},
+                    content_hashes={"hash1", "hash2", "hash3"},
+                ),
+                Promotion(
+                    channels={"chan4", "chan5", "chan6"},
+                    content_hashes={"hash4", "hash5", "hash6"},
+                ),
+                Promotion(
+                    channels={"chan7", "chan8", "chan9"},
+                    content_hashes={"hash7", "hash8", "hash9"},
+                ),
+                Promotion(
+                    channels={"chan10", "chan11", "chan12"},
+                    content_hashes={"hash10", "hash11", "hash12"},
+                ),
+                Promotion(
+                    channels={"chan13", "chan14"},
+                    content_hashes={"hash13", "hash14"},
+                ),
+            ],
+            [
+                OpenMergeRequest(
+                    raw=create_autospec(spec=ProjectMergeRequest),
+                    channels={"chan1", "chan2"},
+                    content_hashes={"hash1", "hash2"},
+                    failed_mr_check=False,
+                    is_batchable=True,
+                ),
+            ],
+            Diff(
+                deletions=[
+                    Deletion(
+                        mr=OpenMergeRequest(
+                            raw=create_autospec(spec=ProjectMergeRequest),
+                            channels={"chan1", "chan2"},
+                            content_hashes={"hash1", "hash2"},
+                            failed_mr_check=False,
+                            is_batchable=True,
+                        ),
+                        reason=MSG_NEW_BATCH,
+                    )
+                ],
+                additions=[
+                    Addition(
+                        # Note, that we expect more than 5 hashes here. The reason is that the
+                        # aggregated promotions consist of 3 hashes, i.e., 3+3 = 6
+                        # This is expected and fine.
+                        content_hashes={
+                            "hash1",
+                            "hash2",
+                            "hash3",
+                            "hash4",
+                            "hash5",
+                            "hash6",
+                        },
+                        channels={"chan1", "chan2", "chan3", "chan4", "chan5", "chan6"},
+                        batchable=True,
+                    ),
+                    Addition(
+                        content_hashes={
+                            "hash7",
+                            "hash8",
+                            "hash9",
+                            "hash10",
+                            "hash11",
+                            "hash12",
+                        },
+                        channels={
+                            "chan7",
+                            "chan8",
+                            "chan9",
+                            "chan10",
+                            "chan11",
+                            "chan12",
+                        },
+                        batchable=True,
+                    ),
+                    Addition(
+                        content_hashes={"hash13", "hash14"},
+                        channels={"chan13", "chan14"},
+                        batchable=True,
+                    ),
+                ],
+            ),
+        ),
+    ],
+)
+def test_reconcile(
+    desired_promotions: list[Promotion],
+    open_mrs: list[OpenMergeRequest],
+    expected_diff: Diff,
+) -> None:
+    reconciler = Reconciler()
+    diff = reconciler.reconcile(
+        desired_promotions=desired_promotions, open_mrs=open_mrs, batch_limit=5
+    )
+
+    # We do not care about the order. As the batcher is working on sets, this
+    # is not deterministic
+    assert len(diff.additions) == len(expected_diff.additions)
+    assert _aggregate_hashes(diff.additions) == _aggregate_hashes(
+        expected_diff.additions
+    )
+    assert _aggregate_channels(diff.additions) == _aggregate_channels(
+        expected_diff.additions
+    )
+
+    # Deletions are actually processed in a deterministic way.
+    assert len(diff.deletions) == len(expected_diff.deletions)
+    for i in range(len(diff.deletions)):
+        assert diff.deletions[i].mr.channels == expected_diff.deletions[i].mr.channels
+        assert (
+            diff.deletions[i].mr.content_hashes
+            == expected_diff.deletions[i].mr.content_hashes
+        )
+        assert (
+            diff.deletions[i].mr.failed_mr_check
+            == expected_diff.deletions[i].mr.failed_mr_check
+        )
+        assert (
+            diff.deletions[i].mr.is_batchable
+            == expected_diff.deletions[i].mr.is_batchable
+        )
+        assert diff.deletions[i].reason == expected_diff.deletions[i].reason

reconcile/test/saas_auto_promotions_manager/test_integration_test.py

@@ -1,124 +1,59 @@
-from collections.abc import (
-    Callable,
-    Iterable,
-    Mapping,
-)
 from unittest.mock import create_autospec
 
 from reconcile.saas_auto_promotions_manager.integration import SaasAutoPromotionsManager
 from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager import (
     MergeRequestManager,
 )
+from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager_v2 import (
+    MergeRequestManagerV2,
+)
 from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
     MRParser,
 )
+from reconcile.saas_auto_promotions_manager.merge_request_manager.reconciler import (
+    Reconciler,
+)
 from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
     Renderer,
 )
 from reconcile.saas_auto_promotions_manager.utils.saas_files_inventory import (
     SaasFilesInventory,
 )
-from reconcile.typed_queries.saas_files import SaasFile
 from reconcile.utils.promotion_state import (
-    PromotionData,
     PromotionState,
 )
 from reconcile.utils.vcs import VCS
 
 
-def test_integration_test(
-    saas_files_builder: Callable[[Iterable[Mapping]], list[SaasFile]],
-    vcs_builder: Callable[..., VCS],
-    promotion_state_builder: Callable[..., PromotionState],
-):
+def test_integration_test():
     """
     Have all the parts glued together and have one full run.
-    This is too complex to setup and maintain for multiple
-    test cases. However, it is a good single test to see if
+    This is too complex to setup and properly maintain.
+    However, it is a good single test to see if
     all components are wired properly.
-
-    These saas files and states should result in a single
-    merge request being opened.
     """
-    saas_files = saas_files_builder([
-        {
-            "path": "/saas1.yml",
-            "name": "saas_1",
-            "resourceTemplates": [
-                {
-                    "name": "template_1",
-                    "url": "repo1/url",
-                    "targets": [
-                        {
-                            "ref": "main",
-                            "namespace": {"path": "/namespace1.yml"},
-                            "promotion": {
-                                "publish": ["channel-1"],
-                            },
-                        },
-                        {
-                            "ref": "main",
-                            "namespace": {"path": "/namespace2.yml"},
-                            "promotion": {
-                                "publish": ["channel-2"],
-                            },
-                        },
-                    ],
-                }
-            ],
-        },
-        {
-            "path": "/saas2.yml",
-            "name": "saas_2",
-            "resourceTemplates": [
-                {
-                    "name": "template_2",
-                    "url": "repo1/url",
-                    "targets": [
-                        {
-                            "ref": "current_sha",
-                            "namespace": {"path": "/namespace3.yml"},
-                            "promotion": {
-                                "subscribe": ["channel-1", "channel-2"],
-                                "auto": True,
-                            },
-                        }
-                    ],
-                }
-            ],
-        },
-    ])
-    vcs = vcs_builder()
-    deployment_state = promotion_state_builder([
-        PromotionData(
-            success=True,
-            target_config_hash="new_hash",
-            saas_file="saas_1",
-        ),
-        PromotionData(
-            success=True,
-            target_config_hash="new_hash",
-            saas_file="saas_1",
-        ),
-    ])
-    renderer = create_autospec(spec=Renderer)
-    mr_parser = create_autospec(spec=MRParser)
+    vcs = create_autospec(spec=VCS)
+    merge_request_manager_v2 = MergeRequestManagerV2(
+        vcs=vcs,
+        reconciler=create_autospec(spec=Reconciler),
+        mr_parser=create_autospec(spec=MRParser),
+        renderer=create_autospec(spec=Renderer),
+    )
     merge_request_manager = MergeRequestManager(
         vcs=vcs,
-        mr_parser=mr_parser,
-        renderer=renderer,
+        mr_parser=create_autospec(spec=MRParser),
+        renderer=create_autospec(spec=Renderer),
     )
     manager = SaasAutoPromotionsManager(
-        deployment_state=deployment_state,
-        saas_file_inventory=SaasFilesInventory(saas_files=saas_files),
+        deployment_state=create_autospec(spec=PromotionState),
+        saas_file_inventory=SaasFilesInventory(saas_files=[]),
         vcs=vcs,
+        merge_request_manager_v2=merge_request_manager_v2,
         merge_request_manager=merge_request_manager,
         thread_pool_size=1,
         dry_run=False,
     )
     manager.reconcile()
 
-    # Only one MR should be opened
-    # TODO: assert MR data
-    vcs.close_app_interface_mr.assert_not_called()  # type: ignore[attr-defined]
-    vcs.open_app_interface_merge_request.assert_called_once()  # type: ignore[attr-defined]
+    vcs.close_app_interface_mr.assert_not_called()
+    vcs.open_app_interface_merge_request.assert_not_called()

reconcile/test/saas_auto_promotions_manager/merge_request_manager/merge_request_manager/test_unbatching.py

@@ -1,96 +0,0 @@
-from collections.abc import (
-    Callable,
-    Iterable,
-)
-from unittest.mock import call, create_autospec
-
-from gitlab.v4.objects import ProjectMergeRequest
-
-from reconcile.saas_auto_promotions_manager.merge_request_manager.merge_request_manager import (
-    MergeRequestManager,
-)
-from reconcile.saas_auto_promotions_manager.merge_request_manager.mr_parser import (
-    MRParser,
-    OpenMergeRequest,
-)
-from reconcile.saas_auto_promotions_manager.merge_request_manager.renderer import (
-    Renderer,
-)
-from reconcile.utils.vcs import VCS
-
-
-def test_unbatch_multiple_valid(
-    mr_parser_builder: Callable[[Iterable], MRParser],
-    renderer: Renderer,
-) -> None:
-    # We have 2 MRs that failed MR check, but are marked as batchable
-    open_mrs = [
-        OpenMergeRequest(
-            raw=create_autospec(spec=ProjectMergeRequest),
-            channels="multiple,aggregated,channels",
-            content_hashes="a,b,c",
-            is_batchable=True,
-            failed_mr_check=True,
-        ),
-        OpenMergeRequest(
-            raw=create_autospec(spec=ProjectMergeRequest),
-            channels="chan",
-            content_hashes="d",
-            is_batchable=True,
-            failed_mr_check=True,
-        ),
-    ]
-    mr_parser = mr_parser_builder(open_mrs)
-
-    vcs = create_autospec(spec=VCS)
-    merge_request_manager = MergeRequestManager(
-        vcs=vcs,
-        mr_parser=mr_parser,
-        renderer=renderer,
-    )
-    expected_close_mr_calls = [
-        call(
-            mr.raw,
-            "Closing this MR because it failed MR check and isn't marked un-batchable yet.",
-        )
-        for mr in open_mrs
-    ]
-    merge_request_manager.housekeeping()
-    vcs.close_app_interface_mr.assert_has_calls(expected_close_mr_calls, any_order=True)
-    assert vcs.close_app_interface_mr.call_count == 2
-    assert merge_request_manager._unbatchable_hashes == set(["a", "b", "c", "d"])
-    assert len(merge_request_manager._open_mrs) == 0
-
-
-def test_unbatch_multiple_invalid(
-    mr_parser_builder: Callable[[Iterable], MRParser],
-    renderer: Renderer,
-) -> None:
-    open_mrs = [
-        OpenMergeRequest(
-            raw=create_autospec(spec=ProjectMergeRequest),
-            channels="multiple,aggregated,channels",
-            content_hashes="a,b,c",
-            is_batchable=False,
-            failed_mr_check=False,
-        ),
-        OpenMergeRequest(
-            raw=create_autospec(spec=ProjectMergeRequest),
-            channels="chan",
-            content_hashes="d",
-            is_batchable=True,
-            failed_mr_check=False,
-        ),
-    ]
-    mr_parser = mr_parser_builder(open_mrs)
-
-    vcs = create_autospec(spec=VCS)
-    merge_request_manager = MergeRequestManager(
-        vcs=vcs,
-        mr_parser=mr_parser,
-        renderer=renderer,
-    )
-    merge_request_manager.housekeeping()
-    vcs.close_app_interface_mr.assert_not_called()
-    assert merge_request_manager._unbatchable_hashes == set()
-    assert len(merge_request_manager._open_mrs) == 2