qontract-reconcile 0.10.1rc1179__py3-none-any.whl → 0.10.1rc1181__py3-none-any.whl

{qontract_reconcile-0.10.1rc1179.dist-info → qontract_reconcile-0.10.1rc1181.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1179
+Version: 0.10.1rc1181
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1179.dist-info → qontract_reconcile-0.10.1rc1181.dist-info}/RECORD

@@ -32,7 +32,7 @@ reconcile/github_validator.py,sha256=cVTVxJIGR4a1Jz8wrdXEAb_CMpXUzvykVmUURX4cook
 reconcile/gitlab_fork_compliance.py,sha256=c7UfqSAsW04c1bWJmXXaQDwtUcG4Kb6nCJAyRU2uAuw,4449
 reconcile/gitlab_housekeeping.py,sha256=D0DOqC-xuMBMct04_MI8Lq32OAi_QMvvGLOz_E-77Dw,22482
 reconcile/gitlab_labeler.py,sha256=4xJHmVX155fclrHqkR926sL1GH6RTN5XfZ8PnqNXbRA,4534
-reconcile/gitlab_members.py,sha256=PrJE9OhDRdGG_gHM_77nQojLb4B18jtUu8DxgLsRS88,8417
+reconcile/gitlab_members.py,sha256=MUIgYDLeJx2-_vMypyq2Pa17cpKdXATYhtVACS2ghpQ,8297
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
 reconcile/gitlab_owners.py,sha256=sn9njaKOtqcvnhi2qtm-faAfAR4zNqflbSuusA9RUuI,13456
 reconcile/gitlab_permissions.py,sha256=6ZBPnQci4-1LVJBvaUS-c0QwIJjITNKVflCeH-Y5Lbk,7507
@@ -112,7 +112,7 @@ reconcile/terraform_aws_route53.py,sha256=dQzzT46YhwRA902_H6pi-f7WlX4EaH187wXSdm
 reconcile/terraform_cloudflare_dns.py,sha256=-aLEe2QnH5cJPu7HWqs-R9NmQ1NlFbcVUm0v7alVL3I,13431
 reconcile/terraform_cloudflare_resources.py,sha256=pq8Ieo5NmB-dYQ9X2F0s6iEoINMzhiqGw2yQK4ovok4,14980
 reconcile/terraform_cloudflare_users.py,sha256=iyTG5sj20Jg4J4qWJ144KVptfIHGOSfH8wQKxu0imq0,13942
-reconcile/terraform_repo.py,sha256=TKqlodhQGoAtQ6nDm04TNlpx4wpgJ_n4atoUK5Rfd7o,16444
+reconcile/terraform_repo.py,sha256=jT8zD-V1_5D6yB2gLYP1nEC9wHoHktDQYnJqpo2EC9M,15807
 reconcile/terraform_resources.py,sha256=iufjMJs_aSEvmh7Cg11beCxKmV8nrOLOpEtiTryPNx0,19470
 reconcile/terraform_tgw_attachments.py,sha256=09svJG9pAiwWp4aY0xRoQRV90T4ZNwHG3r8flI-ZS_s,18810
 reconcile/terraform_users.py,sha256=HqSm3ev3b8dZ9J6F_phDZB-FQsnlsdeKp9RPoY1cU94,10188
@@ -520,7 +520,7 @@ reconcile/test/test_github_org.py,sha256=aGWeMhkz1fjogvaAQGjemSKR201L5gEZZOe0iMQ
 reconcile/test/test_github_repo_invites.py,sha256=WrPn5ROAoJYK0ihzlZcFR0V9Pu2HcMs13I6nazf7hq4,3307
 reconcile/test/test_gitlab_housekeeping.py,sha256=ScD9Tgf9OOgGfAFfTy6Kn2222l2wH_A3gMRKdpvoWe0,10053
 reconcile/test/test_gitlab_labeler.py,sha256=PmYXiU2g0_O5OTdMGPzdeqBAfat92U9bhjjfeyiGSmQ,4336
-reconcile/test/test_gitlab_members.py,sha256=yjfQRUFG_F0kLYegax4_ec5VIBAnCPrvAgqMcN1GXzc,9985
+reconcile/test/test_gitlab_members.py,sha256=bupl1dsLor-913LGYCuRtJMKqDsXo_d_2nbtDe6B1Us,7016
 reconcile/test/test_gitlab_permissions.py,sha256=aMf5SUeVp-aQ1bWGQPQLYa85auzRlyfZVTWqyybJ6mo,5850
 reconcile/test/test_instrumented_wrappers.py,sha256=CZzhnQH0c4i7-Rxjg7-0dfFMvVPegLHL46z5NHOOCwo,608
 reconcile/test/test_integrations_manager.py,sha256=xpyQAVz57wAbovrcQzAeuyq8VzdItUyW2d2kp1WW_5c,38184
@@ -566,7 +566,7 @@ reconcile/test/test_terraform_aws_route53.py,sha256=xHggb8K1P76OyCfFcogbkmyKle-N
 reconcile/test/test_terraform_cloudflare_dns.py,sha256=aQTXX8Vr4h9aWvJZTnpZEhMGYoBpT2d45ZxU_ECIQ6o,3425
 reconcile/test/test_terraform_cloudflare_resources.py,sha256=1mdSZS-38mtTSg7teJgDCJU1b_yKbwnrr3N5m8kwV4k,13595
 reconcile/test/test_terraform_cloudflare_users.py,sha256=2VGBtMUhckLPtUnQlHIzpGsCnyVJZPNLFf-ABELkxbQ,27456
-reconcile/test/test_terraform_repo.py,sha256=Th6MNbexZX-pTtEpYFiXX0cMAZfnuFcYQ71ZZSjumbE,13025
+reconcile/test/test_terraform_repo.py,sha256=r6bwryHtq6-VuUh3fuMsxlsjbQAgrcQUHOEFou_hilQ,12260
 reconcile/test/test_terraform_resources.py,sha256=8C97yXIEihaQ3DZrtjxLNt4y4G12IOhD01ydm7JjliY,15359
 reconcile/test/test_terraform_tgw_attachments.py,sha256=pJFmQzUwAn-wKpF6oSkImpdF7WXvcky8iaJiXbjGGgU,41104
 reconcile/test/test_terraform_users.py,sha256=XOAfGvITCJPI1LTlISmHbA4ONMQMkxYUMTsny7pQCFw,4319
@@ -678,7 +678,7 @@ reconcile/utils/external_resources.py,sha256=y7Wz32cOAmCsUhQ6T-1N6lktnLikGkaHQ0S
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=wzVIYAeKlMGW538U1mkJWUI6h_mFRUY4lawh2AR8hw4,2345
 reconcile/utils/github_api.py,sha256=R8OvqyPdnRqvP-Efnv9RvIcbBlb4M0KC4RlbnJMD0Tg,2426
-reconcile/utils/gitlab_api.py,sha256=C1nsHQKKybsmFdaG9vsItBjJm69ym4VWbqbKfAEf7oY,29305
+reconcile/utils/gitlab_api.py,sha256=ar3D1gaPGm71ecQReMvbMbBzCa8TRs3Pn1ZZJP_lwSw,28453
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -880,8 +880,8 @@ tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6d
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1179.dist-info/METADATA,sha256=TrEzA9ztxHZUkVWE9KPoeEMIg_cEtRmS-JDoLYPzeHo,2213
-qontract_reconcile-0.10.1rc1179.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-qontract_reconcile-0.10.1rc1179.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1179.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1179.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1181.dist-info/METADATA,sha256=pDZwlb7DGvqkX4trR3vqj0NqCsBK-LPoCvo46Uufung,2213
+qontract_reconcile-0.10.1rc1181.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+qontract_reconcile-0.10.1rc1181.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1181.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1181.dist-info/RECORD,,

reconcile/gitlab_members.py

@@ -1,8 +1,11 @@
-import enum
 import logging
 from collections.abc import Callable
 from typing import Any
 
+from gitlab.v4.objects import (
+    Group,
+    GroupMember,
+)
 from pydantic import BaseModel
 
 from reconcile import queries
@@ -23,6 +26,7 @@ from reconcile.gql_definitions.gitlab_members.permissions import (
 )
 from reconcile.utils import gql
 from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.exceptions import AppInterfaceSettingsError
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.pagerduty_api import (
@@ -37,50 +41,51 @@ QONTRACT_INTEGRATION = "gitlab-members"
 
 class GitlabUser(BaseModel):
     user: str
-    access_level: str
+    access_level: int
 
 
-State = dict[str, list[GitlabUser]]
+class CurrentStateSpec(BaseModel):
+    members: dict[str, GroupMember]
 
+    class Config:
+        arbitrary_types_allowed = True
 
-class Action(enum.Enum):
-    add_user_to_group = enum.auto()
-    remove_user_from_group = enum.auto()
-    change_access = enum.auto()
 
+class DesiredStateSpec(BaseModel):
+    members: dict[str, GitlabUser]
+
+    class Config:
+        arbitrary_types_allowed = True
 
-class Diff(BaseModel):
-    action: Action
-    group: str
-    user: str
-    access_level: str
 
+CurrentState = dict[str, CurrentStateSpec]
+DesiredState = dict[str, DesiredStateSpec]
 
-def get_current_state(instance: GitlabInstanceV1, gl: GitLabApi) -> State:
+
+def get_current_state(
+    instance: GitlabInstanceV1, gl: GitLabApi, gitlab_groups_map: dict[str, Group]
+) -> CurrentState:
     """Get current gitlab group members for all managed groups."""
     return {
-        g: [
-            GitlabUser(user=u["user"], access_level=u["access_level"])
-            for u in gl.get_group_members(g)
-        ]
+        g: CurrentStateSpec(
+            members={
+                u.username: u for u in gl.get_group_members(gitlab_groups_map.get(g))
+            },
+        )
         for g in instance.managed_groups
     }
 
 
 def add_or_update_user(
-    group_members: State, group_name: str, gitlab_user: GitlabUser
+    desired_state_spec: DesiredStateSpec, gitlab_user: GitlabUser
 ) -> None:
-    existing_users = [
-        gu for gu in group_members[group_name] if gu.user == gitlab_user.user
-    ]
-    if not existing_users:
-        group_members[group_name].append(gitlab_user)
+    existing_user = desired_state_spec.members.get(gitlab_user.user)
+    if not existing_user:
+        desired_state_spec.members[gitlab_user.user] = gitlab_user
     else:
-        existing_user = existing_users[0]
-        if GitLabApi.get_access_level(
-            existing_user.access_level
-        ) < GitLabApi.get_access_level(gitlab_user.access_level):
-            existing_user.access_level = gitlab_user.access_level
+        existing_user.access_level = max(
+            existing_user.access_level, gitlab_user.access_level
+        )
 
 
 def get_desired_state(
@@ -88,95 +93,41 @@ def get_desired_state(
     pagerduty_map: PagerDutyMap,
     permissions: list[PermissionGitlabGroupMembershipV1],
     all_users: list[User],
-) -> State:
+) -> DesiredState:
     """Fetch all desired gitlab users from app-interface."""
-    desired_group_members: State = {g: [] for g in instance.managed_groups}
-    for g in desired_group_members:
-        for p in permissions:
-            if p.group == g:
-                for r in p.roles or []:
-                    for u in (r.users or []) + (r.bots or []):
-                        gu = GitlabUser(user=u.org_username, access_level=p.access)
-                        add_or_update_user(desired_group_members, g, gu)
-                if p.pagerduty:
-                    usernames_from_pagerduty = get_usernames_from_pagerduty(
-                        p.pagerduty,
-                        all_users,
-                        g,
-                        pagerduty_map,
-                        get_username_method=lambda u: u.org_username,
-                    )
-                    for u in usernames_from_pagerduty:
-                        gu = GitlabUser(user=u, access_level=p.access)
-                        add_or_update_user(desired_group_members, g, gu)
-
+    desired_group_members: DesiredState = {
+        g: build_desired_state_spec(g, permissions, pagerduty_map, all_users)
+        for g in instance.managed_groups
+    }
     return desired_group_members
 
 
-def calculate_diff(current_state: State, desired_state: State) -> list[Diff]:
-    """Compare current and desired state and return all differences."""
-    diff: list[Diff] = []
-    diff += subtract_states(desired_state, current_state, Action.add_user_to_group)
-    diff += subtract_states(current_state, desired_state, Action.remove_user_from_group)
-    diff += check_access(current_state, desired_state)
-    return diff
-
-
-def subtract_states(
-    from_state: State, subtract_state: State, action: Action
-) -> list[Diff]:
-    """Return diff objects for items in from_state but not in subtract_state."""
-    result = []
-    for f_group, f_users in from_state.items():
-        s_group = subtract_state[f_group]
-        for f_user in f_users:
-            found = False
-            for s_user in s_group:
-                if f_user.user != s_user.user:
-                    continue
-                found = True
-                break
-            if not found:
-                result.append(
-                    Diff(
-                        action=action,
-                        group=f_group,
-                        user=f_user.user,
-                        access_level=f_user.access_level,
-                    )
+def build_desired_state_spec(
+    group_name: str,
+    permissions: list[PermissionGitlabGroupMembershipV1],
+    pagerduty_map: PagerDutyMap,
+    all_users: list[User],
+) -> DesiredStateSpec:
+    desired_state_spec = DesiredStateSpec(members={})
+    for p in permissions:
+        if p.group == group_name:
+            p_access_level = GitLabApi.get_access_level(p.access)
+            for r in p.roles or []:
+                for u in (r.users or []) + (r.bots or []):
+                    gu = GitlabUser(user=u.org_username, access_level=p_access_level)
+                    add_or_update_user(desired_state_spec, gu)
+            if p.pagerduty:
+                usernames_from_pagerduty = get_usernames_from_pagerduty(
+                    p.pagerduty,
+                    all_users,
+                    group_name,
+                    pagerduty_map,
+                    get_username_method=lambda u: u.org_username,
                 )
-    return result
-
-
-def check_access(current_state: State, desired_state: State) -> list[Diff]:
-    """Return diff objects for item where access level is different."""
-    result = []
-    for d_group, d_users in desired_state.items():
-        c_group = current_state[d_group]
-        for d_user in d_users:
-            for c_user in c_group:
-                if d_user.user == c_user.user:
-                    if d_user.access_level != c_user.access_level:
-                        result.append(
-                            Diff(
-                                action=Action.change_access,
-                                group=d_group,
-                                user=c_user.user,
-                                access_level=d_user.access_level,
-                            )
-                        )
-                    break
-    return result
-
-
-def act(diff: Diff, gl: GitLabApi) -> None:
-    """Apply a diff object."""
-    if diff.action == Action.remove_user_from_group:
-        gl.remove_group_member(diff.group, diff.user)
-    if diff.action == Action.add_user_to_group:
-        gl.add_group_member(diff.group, diff.user, diff.access_level)
-    if diff.action == Action.change_access:
-        gl.change_access(diff.group, diff.user, diff.access_level)
+                for u in usernames_from_pagerduty:
+                    gu = GitlabUser(user=u, access_level=p_access_level)
+                    add_or_update_user(desired_state_spec, gu)
+    return desired_state_spec
 
 
 def get_permissions(query_func: Callable) -> list[PermissionGitlabGroupMembershipV1]:
@@ -197,6 +148,11 @@ def get_gitlab_instance(query_func: Callable) -> GitlabInstanceV1:
     raise AppInterfaceSettingsError("No gitlab instance found!")
 
 
+def get_managed_groups_map(group_names: list[str], gl: GitLabApi) -> dict[str, Group]:
+    gitlab_groups = {group_name: gl.get_group(group_name) for group_name in group_names}
+    return gitlab_groups
+
+
 @defer
 def run(
     dry_run: bool,
@@ -228,17 +184,58 @@ def run(
     pagerduty_map = get_pagerduty_map(
         secret_reader, pagerduty_instances=pagerduty_instances
     )
-
-    # act
-    current_state = get_current_state(instance, gl)
+    managed_groups_map = get_managed_groups_map(instance.managed_groups, gl)
+    current_state = get_current_state(instance, gl, managed_groups_map)
     desired_state = get_desired_state(instance, pagerduty_map, permissions, all_users)
-    diffs = calculate_diff(current_state, desired_state)
-
-    for diff in diffs:
-        logging.info(diff)
-
-        if not dry_run:
-            act(diff, gl)
+    for group_name in instance.managed_groups:
+        reconcile_gitlab_members(
+            current_state_spec=current_state.get(group_name),
+            desired_state_spec=desired_state.get(group_name),
+            group=managed_groups_map.get(group_name),
+            gl=gl,
+            dry_run=dry_run,
+        )
+
+
+def reconcile_gitlab_members(
+    current_state_spec: CurrentStateSpec | None,
+    desired_state_spec: DesiredStateSpec | None,
+    group: Group | None,
+    gl: GitLabApi,
+    dry_run: bool,
+) -> None:
+    if current_state_spec and desired_state_spec and group:
+        diff_data = diff_mappings(
+            current=current_state_spec.members,
+            desired=desired_state_spec.members,
+            equal=lambda current, desired: current.access_level == desired.access_level,
+        )
+        for key, gitlab_user in diff_data.add.items():
+            logging.info([
+                key,
+                "add_user_to_group",
+                group.name,
+                gl.get_access_level_string(gitlab_user.access_level),
+            ])
+            if not dry_run:
+                gl.add_group_member(group, gitlab_user)
+        for key, group_member in diff_data.delete.items():
+            logging.info([
+                key,
+                "remove_user_from_group",
+                group.name,
+            ])
+            if not dry_run:
+                gl.remove_group_member(group, group_member.id)
+        for key, diff_pair in diff_data.change.items():
+            logging.info([
+                key,
+                "change_access",
+                group.name,
+                gl.get_access_level_string(diff_pair.desired.access_level),
+            ])
+            if not dry_run:
+                gl.change_access(diff_pair.current, diff_pair.desired.access_level)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:

reconcile/terraform_repo.py

@@ -100,7 +100,6 @@ class TerraformRepoIntegration(
                 desired_state=desired,
                 dry_run=dry_run,
                 state=state,
-                recreate_state=False,
             )
 
             if repo_diff_result:
@@ -110,18 +109,19 @@ class TerraformRepoIntegration(
             # the existing state stored in S3. This is due to a behavior in Pydantic V1 that has since been addressed in V2
             # https://docs.pydantic.dev/latest/blog/pydantic-v2/#required-vs-nullable-cleanup
             # so in this case, tf-repo will just recreate all of those state files in S3 and not actually do a plan or apply
-            logging.error(err)
-            logging.info(
-                "Unable to parse existing Terraform-Repo state from S3. Note that this is separate from the actual .tfstate files. Terraform Repo will re-create its own state upon merge and will not update any infrastructure. This typically occurs with changes to the Terraform Repo schema files and is normally resolved once state is re-created."
+            logging.warning(
+                "Unable to parse existing Terraform-Repo state from S3. Will re-create internal state upon apply, more info here: https://gitlab.cee.redhat.com/service/app-interface/-/blob/master/docs/terraform-repo/sop/statefile-load-errors.md",
+                exc_info=err,
             )
             repo_diff_result = self.calculate_diff(
                 existing_state=[],
                 desired_state=desired,
                 dry_run=dry_run,
                 state=state,
-                recreate_state=True,
             )
 
+            # we don't call print_output here meaning that the executor will not plan/apply
+
     def print_output(self, diff: list[TerraformRepoV1], dry_run: bool) -> OutputFile:
         """Parses and prints the output of a Terraform Repo diff for the executor
 
@@ -302,7 +302,6 @@ class TerraformRepoIntegration(
         desired_state: list[TerraformRepoV1],
         dry_run: bool,
         state: State | None,
-        recreate_state: bool,
     ) -> list[TerraformRepoV1] | None:
         """Calculated the difference between existing and desired state
         to determine what actions the executor will need to take
@@ -315,8 +314,6 @@ class TerraformRepoIntegration(
         :type dry_run: bool
         :param state: AWS S3 state
         :type state: Optional[State]
-        :param recreate_state: whether we are recreating our own state
-        :type recreate_state: bool
         :raises ParameterError: if there is an invalid operation performed like trying to delete
         a representation in A-I before setting the delete flag
         :return: the terraform repo to act on
@@ -326,13 +323,6 @@ class TerraformRepoIntegration(
 
         merged = self.merge_results(diff)
 
-        # validate that only one repo is being modified in each MR
-        # this lets us fail early and avoid multiple GL requests we don't need to make
-        if dry_run and len(merged) > 1 and not recreate_state:
-            raise Exception(
-                "Only one repository can be modified per merge request, please split your change out into multiple MRs. Hint: try rebasing your merge request"
-            )
-
         # added repos: do standard validation that SHA is valid
         if self.params.validate_git:
             for add_repo in diff.add.values():

reconcile/test/test_gitlab_members.py

@@ -1,15 +1,20 @@
-import copy
 from typing import Any
+from unittest.mock import create_autospec
 
 import pytest
+from gitlab.v4.objects import (
+    Group,
+    GroupMember,
+)
 from pytest_mock import MockerFixture
 
 from reconcile import gitlab_members
 from reconcile.gitlab_members import (
-    Action,
-    Diff,
+    CurrentState,
+    CurrentStateSpec,
+    DesiredState,
+    DesiredStateSpec,
     GitlabUser,
-    State,
     add_or_update_user,
     get_permissions,
 )
@@ -37,16 +42,41 @@ def instance(vault_secret: VaultSecret) -> GitlabInstanceV1:
 
 
 @pytest.fixture()
-def state() -> State:
+def gitlab_groups_map() -> dict[str, Group]:
+    group1 = create_autospec(Group, name="group1", id="123")
+    group1.name = "group1"
+    group2 = create_autospec(Group, name="group2", id="124")
+    group2.name = "group2"
     return {
-        "group1": [
-            GitlabUser(access_level="developer", user="user1"),
-            GitlabUser(access_level="maintainer", user="user2"),
-        ],
-        "group2": [
-            GitlabUser(access_level="developer", user="user3"),
-            GitlabUser(access_level="maintainer", user="user4"),
-        ],
+        "group1": group1,
+        "group2": group2,
+    }
+
+
+@pytest.fixture()
+def all_users() -> list[GroupMember]:
+    user1 = create_autospec(GroupMember, username="user1", id="123", access_level=30)
+    user2 = create_autospec(GroupMember, username="user2", id="124", access_level=40)
+    user3 = create_autospec(GroupMember, username="user3", id="125", access_level=30)
+    user4 = create_autospec(GroupMember, username="user4", id="126", access_level=40)
+    return [user1, user2, user3, user4]
+
+
+@pytest.fixture()
+def current_state(all_users: list[GroupMember]) -> CurrentState:
+    return {
+        "group1": CurrentStateSpec(
+            members={
+                "user1": all_users[0],
+                "user2": all_users[1],
+            },
+        ),
+        "group2": CurrentStateSpec(
+            members={
+                "user3": all_users[2],
+                "user4": all_users[3],
+            },
+        ),
     }
 
 
@@ -72,20 +102,27 @@ def user() -> User:
 
 
 def test_gitlab_members_get_current_state(
-    mocker: MockerFixture, instance: GitlabInstanceV1, state: State
+    mocker: MockerFixture,
+    instance: GitlabInstanceV1,
+    current_state: CurrentState,
+    gitlab_groups_map: dict[str, Group],
+    all_users: list[GroupMember],
 ) -> None:
     gl_mock = mocker.create_autospec(GitLabApi)
     gl_mock.get_group_members.side_effect = [
         [
-            {"user": "user1", "access_level": "developer"},
-            {"user": "user2", "access_level": "maintainer"},
+            all_users[0],
+            all_users[1],
         ],
         [
-            {"user": "user3", "access_level": "developer"},
-            {"user": "user4", "access_level": "maintainer"},
+            all_users[2],
+            all_users[3],
         ],
     ]
-    assert gitlab_members.get_current_state(instance, gl_mock) == state
+    assert (
+        gitlab_members.get_current_state(instance, gl_mock, gitlab_groups_map)
+        == current_state
+    )
 
 
 def test_gitlab_members_get_desired_state(
@@ -103,210 +140,80 @@ def test_gitlab_members_get_desired_state(
     assert gitlab_members.get_desired_state(
         instance, mock_pagerduty_map, permissions, [user]
     ) == {
-        "group1": [GitlabUser(user="devtools-bot", access_level="owner")],
-        "group2": [
-            GitlabUser(user="devtools-bot", access_level="owner"),
-            GitlabUser(user="user1", access_level="owner"),
-            GitlabUser(user="user2", access_level="owner"),
-            GitlabUser(user="user3", access_level="owner"),
-            GitlabUser(user="user4", access_level="owner"),
-            GitlabUser(user="another-bot", access_level="owner"),
-        ],
-    }
-
-
-def test_gitlab_members_calculate_diff_no_changes(state: State) -> None:
-    # pylint: disable-next=use-implicit-booleaness-not-comparison # for better readability
-    assert gitlab_members.calculate_diff(state, state) == []
-
-
-def test_gitlab_members_subtract_states_no_changes_add(state: State) -> None:
-    # pylint: disable-next=use-implicit-booleaness-not-comparison # for better readability
-    assert gitlab_members.subtract_states(state, state, Action.add_user_to_group) == []
-
-
-def test_gitlab_members_subtract_states_no_changes_remove(state: State) -> None:
-    # pylint: disable=use-implicit-booleaness-not-comparison # for better readability
-    assert (
-        gitlab_members.subtract_states(state, state, Action.remove_user_from_group)
-        == []
-    )
-
-
-def test_gitlab_members_subtract_states_add(state: State) -> None:
-    current_state = copy.deepcopy(state)
-    # enforce add users to groups
-    current_state["group2"] = [GitlabUser(access_level="maintainer", user="otherone")]
-    del current_state["group1"][1]
-
-    desired_state = state
-    assert gitlab_members.subtract_states(
-        desired_state, current_state, Action.add_user_to_group
-    ) == [
-        Diff(
-            action=Action.add_user_to_group,
-            group="group1",
-            user="user2",
-            access_level="maintainer",
-        ),
-        Diff(
-            action=Action.add_user_to_group,
-            group="group2",
-            user="user3",
-            access_level="developer",
-        ),
-        Diff(
-            action=Action.add_user_to_group,
-            group="group2",
-            user="user4",
-            access_level="maintainer",
-        ),
-    ]
-
-
-def test_gitlab_members_subtract_states_remove(state: State) -> None:
-    current_state = copy.deepcopy(state)
-    # enforce remove user from group
-    current_state["group2"] = [GitlabUser(access_level="maintainer", user="otherone")]
-
-    desired_state = state
-    assert gitlab_members.subtract_states(
-        current_state, desired_state, Action.remove_user_from_group
-    ) == [
-        Diff(
-            action=Action.remove_user_from_group,
-            group="group2",
-            user="otherone",
-            access_level="maintainer",
-        )
-    ]
-
-
-def test_gitlab_members_check_access_no_changes(state: State) -> None:
-    # pylint: disable-next=use-implicit-booleaness-not-comparison # for better readability
-    assert gitlab_members.check_access(state, state) == []
-
-
-def test_gitlab_members_check_access(state: State) -> None:
-    current_state = copy.deepcopy(state)
-    # enforce access change
-    current_state["group1"][0].access_level = "owner"
-    desired_state = state
-    assert gitlab_members.check_access(current_state, desired_state) == [
-        Diff(
-            action=Action.change_access,
-            group="group1",
-            user="user1",
-            access_level="developer",
-        ),
-    ]
-
-
-def test_gitlab_members_calculate_diff_changes(state: State) -> None:
-    current_state = copy.deepcopy(state)
-    # enforce remove user from group
-    current_state["group2"] = [GitlabUser(access_level="maintainer", user="otherone")]
-    # enforce add user to group
-    del current_state["group1"][1]
-    # enforce access change
-    current_state["group1"][0].access_level = "owner"
-    desired_state = state
-    assert gitlab_members.calculate_diff(current_state, desired_state) == [
-        Diff(
-            action=Action.add_user_to_group,
-            group="group1",
-            user="user2",
-            access_level="maintainer",
-        ),
-        Diff(
-            action=Action.add_user_to_group,
-            group="group2",
-            user="user3",
-            access_level="developer",
-        ),
-        Diff(
-            action=Action.add_user_to_group,
-            group="group2",
-            user="user4",
-            access_level="maintainer",
-        ),
-        Diff(
-            action=Action.remove_user_from_group,
-            group="group2",
-            user="otherone",
-            access_level="maintainer",
+        "group1": DesiredStateSpec(
+            members={"devtools-bot": GitlabUser(user="devtools-bot", access_level=50)},
         ),
-        Diff(
-            action=Action.change_access,
-            group="group1",
-            user="user1",
-            access_level="developer",
+        "group2": DesiredStateSpec(
+            members={
+                "devtools-bot": GitlabUser(user="devtools-bot", access_level=50),
+                "user1": GitlabUser(user="user1", access_level=50),
+                "user2": GitlabUser(user="user2", access_level=50),
+                "user3": GitlabUser(user="user3", access_level=50),
+                "user4": GitlabUser(user="user4", access_level=50),
+                "another-bot": GitlabUser(user="another-bot", access_level=50),
+            },
         ),
-    ]
-
-
-def test_gitlab_members_act_add(mocker: MockerFixture) -> None:
-    gl_mock = mocker.create_autospec(GitLabApi)
-    diff = Diff(
-        action=Action.add_user_to_group,
-        group="group2",
-        user="user4",
-        access_level="maintainer",
-    )
-    gitlab_members.act(diff, gl_mock)
-    gl_mock.add_group_member.assert_called_once()
-    gl_mock.remove_group_member.assert_not_called()
-    gl_mock.change_access.assert_not_called()
-
-
-def test_gitlab_members_act_remove(mocker: MockerFixture) -> None:
-    gl_mock = mocker.create_autospec(GitLabApi)
-    diff = Diff(
-        action=Action.remove_user_from_group,
-        group="group2",
-        user="otherone",
-        access_level="maintainer",
-    )
-    gitlab_members.act(diff, gl_mock)
-    gl_mock.add_group_member.assert_not_called()
-    gl_mock.remove_group_member.assert_called_once()
-    gl_mock.change_access.assert_not_called()
+    }
 
 
-def test_gitlab_members_act_change(mocker: MockerFixture) -> None:
+def test_gitlab_members_reconcile_gitlab_members(
+    gitlab_groups_map: dict[str, Group],
+    mocker: MockerFixture,
+    all_users: list[GroupMember],
+) -> None:
     gl_mock = mocker.create_autospec(GitLabApi)
-    diff = Diff(
-        action=Action.change_access,
-        group="group1",
-        user="user1",
-        access_level="developer",
+    current_state: CurrentState = {
+        "group1": CurrentStateSpec(
+            members={
+                "user1": all_users[0],
+                "user3": all_users[2],
+                "user4": all_users[3],
+            },
+        )
+    }
+    new_user = GitlabUser(user="new_user", access_level=40)
+    desired_state: DesiredState = {
+        "group1": DesiredStateSpec(
+            members={
+                "user1": GitlabUser(user="user1", access_level=30),
+                "new_user": new_user,
+                "user3": GitlabUser(user="user3", access_level=50),
+            }
+        )
+    }
+    group = gitlab_groups_map.get("group1")
+    gitlab_members.reconcile_gitlab_members(
+        current_state_spec=current_state.get("group1"),
+        desired_state_spec=desired_state.get("group1"),
+        group=group,
+        dry_run=False,
+        gl=gl_mock,
     )
-    gitlab_members.act(diff, gl_mock)
-    gl_mock.add_group_member.assert_not_called()
-    gl_mock.remove_group_member.assert_not_called()
-    gl_mock.change_access.assert_called_once()
+    gl_mock.add_group_member.assert_called_once_with(group, new_user)
+    gl_mock.change_access.assert_called_once_with(all_users[2], 50)
+    gl_mock.remove_group_member.assert_called_once_with(group, all_users[3].id)
 
 
 def test_add_or_update_user_add():
-    group_members: State = {"t": []}
-    gu = GitlabUser(user="u", access_level="owner")
-    add_or_update_user(group_members, "t", gu)
-    assert group_members == {"t": [gu]}
+    desired_state_spec: DesiredStateSpec = DesiredStateSpec(members={})
+    gu = GitlabUser(user="u", access_level=50, id="1234")
+    add_or_update_user(desired_state_spec, gu)
+    assert desired_state_spec == DesiredStateSpec(members={"u": gu})
 
 
 def test_add_or_update_user_update_higher():
-    group_members: State = {"t": []}
-    gu1 = GitlabUser(user="u", access_level="maintainer")
-    gu2 = GitlabUser(user="u", access_level="owner")
-    add_or_update_user(group_members, "t", gu1)
-    add_or_update_user(group_members, "t", gu2)
-    assert group_members == {"t": [gu2]}
+    desired_state_spec: DesiredStateSpec = DesiredStateSpec(members={})
+    gu1 = GitlabUser(user="u", access_level=40)
+    gu2 = GitlabUser(user="u", access_level=50)
+    add_or_update_user(desired_state_spec, gu1)
+    add_or_update_user(desired_state_spec, gu2)
+    assert desired_state_spec == DesiredStateSpec(members={"u": gu2})
 
 
 def test_add_or_update_user_update_lower():
-    group_members: State = {"t": []}
-    gu1 = GitlabUser(user="u", access_level="owner")
-    gu2 = GitlabUser(user="u", access_level="maintainer")
-    add_or_update_user(group_members, "t", gu1)
-    add_or_update_user(group_members, "t", gu2)
-    assert group_members == {"t": [gu1]}
+    desired_state_spec: DesiredStateSpec = DesiredStateSpec(members={})
+    gu1 = GitlabUser(user="u", access_level=50)
+    gu2 = GitlabUser(user="u", access_level=40)
+    add_or_update_user(desired_state_spec, gu1)
+    add_or_update_user(desired_state_spec, gu2)
+    assert desired_state_spec == DesiredStateSpec(members={"u": gu1})

reconcile/test/test_terraform_repo.py

@@ -193,7 +193,6 @@ def test_addition_to_existing_repo(existing_repo, new_repo, int_params, state_mo
         desired_state=desired,
         dry_run=False,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff == [new_repo]
@@ -215,7 +214,6 @@ def test_updating_repo_ref(existing_repo, int_params, state_mock):
         desired_state=[updated_repo],
         dry_run=False,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff == [updated_repo]
@@ -236,7 +234,6 @@ def test_force_rerun(existing_repo, int_params, state_mock):
         desired_state=[updated_repo],
         dry_run=False,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff == [updated_repo]
@@ -263,7 +260,6 @@ def test_fail_on_update_invalid_repo_params(existing_repo, int_params):
             desired_state=[updated_repo],
             dry_run=True,
             state=None,
-            recreate_state=False,
         )
 
 
@@ -279,7 +275,6 @@ def test_delete_repo(existing_repo, int_params, state_mock):
         desired_state=[updated_repo],
         dry_run=False,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff == [updated_repo]
@@ -298,7 +293,6 @@ def test_delete_repo_without_flag(existing_repo, int_params):
             desired_state=[],
             dry_run=True,
             state=None,
-            recreate_state=False,
         )
 
 
@@ -361,7 +355,6 @@ def test_update_repo_state(int_params, existing_repo, state_mock):
         desired_state=desired_state,
         dry_run=False,
         state=state_mock,
-        recreate_state=False,
     )
 
     state_mock.add.assert_called_once_with(
@@ -384,7 +377,6 @@ def test_output_correct_statefile(
         desired_state=desired_state,
         dry_run=True,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff
@@ -406,7 +398,6 @@ def test_output_correct_no_statefile(
         desired_state=desired_state,
         dry_run=True,
         state=state_mock,
-        recreate_state=False,
     )
 
     assert diff
@@ -415,21 +406,6 @@ def test_output_correct_no_statefile(
     assert new_repo_output == current_output
 
 
-def test_fail_on_multiple_repos_dry_run(int_params, existing_repo, new_repo):
-    integration = TerraformRepoIntegration(params=int_params)
-
-    desired_state = [existing_repo, new_repo]
-
-    with pytest.raises(Exception):
-        integration.calculate_diff(
-            existing_state=[],
-            desired_state=desired_state,
-            dry_run=True,
-            state=None,
-            recreate_state=False,
-        )
-
-
 def test_succeed_on_multiple_repos_non_dry_run(int_params, existing_repo, new_repo):
     integration = TerraformRepoIntegration(params=int_params)
 
@@ -440,7 +416,6 @@ def test_succeed_on_multiple_repos_non_dry_run(int_params, existing_repo, new_re
         desired_state=desired_state,
         dry_run=False,
         state=None,
-        recreate_state=False,
     )
 
     assert diff
@@ -460,7 +435,6 @@ def test_no_op_succeeds(int_params, existing_repo):
         desired_state=state,
         dry_run=True,
         state=None,
-        recreate_state=False,
     )
 
     assert diff is None

reconcile/utils/gitlab_api.py

@@ -29,6 +29,7 @@ from gitlab.const import (
 from gitlab.v4.objects import (
     CurrentUser,
     Group,
+    GroupMember,
     PersonalAccessToken,
     Project,
     ProjectIssue,
@@ -84,6 +85,7 @@ GROUP_BOT_NAME_REGEX = re.compile(r"group_.+_bot_.+")
 
 
 class GLGroupMember(TypedDict):
+    id: str
     user: str
     access_level: str
 
@@ -288,17 +290,13 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
         """
         return GROUP_BOT_NAME_REGEX.match(username) is not None
 
-    def get_group_members(self, group_name: str) -> list[GLGroupMember]:
-        group = self.get_group_if_exists(group_name)
+    def get_group_members(self, group: Group | None) -> list[GroupMember]:
         if group is None:
-            logging.error(group_name + " group not found")
+            logging.error("no group provided")
             return []
         else:
             return [
-                {
-                    "user": m.username,
-                    "access_level": self.get_access_level_string(m.access_level),
-                }
+                m
                 for m in self.get_items(group.members.list)
                 if not self._is_bot_username(m.username)
             ]
@@ -315,40 +313,27 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
             member.access_level = access_level
             member.save()
 
-    def add_group_member(self, group_name, username, access):
-        group = self.get_group_if_exists(group_name)
-        if not group:
-            logging.error(group_name + " group not found")
-        else:
-            user = self.get_user(username)
-            access_level = self.get_access_level(access)
-            if user is not None:
-                gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-                try:
-                    group.members.create({
-                        "user_id": user.id,
-                        "access_level": access_level,
-                    })
-                except gitlab.exceptions.GitlabCreateError:
-                    gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-                    member = group.members.get(user.id)
-                    member.access_level = access_level
-
-    def remove_group_member(self, group_name, username):
-        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        group = self.gl.groups.get(group_name)
-        user = self.get_user(username)
-        if user is not None:
+    def add_group_member(self, group, user):
+        gitlab_user = self.get_user(user.user)
+        if gitlab_user is not None:
             gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-            group.members.delete(user.id)
+            try:
+                group.members.create({
+                    "user_id": gitlab_user.id,
+                    "access_level": user.access_level,
+                })
+            except gitlab.exceptions.GitlabCreateError:
+                gitlab_request.labels(integration=INTEGRATION_NAME).inc()
+                member = group.members.get(user.user)
+                member.access_level = user.access_level
+                member.save()
 
-    def change_access(self, group, username, access):
-        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        group = self.gl.groups.get(group)
-        user = self.get_user(username)
+    def remove_group_member(self, group, user_id):
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        member = group.members.get(user.id)
-        member.access_level = self.get_access_level(access)
+        group.members.delete(user_id)
+
+    def change_access(self, member, access_level):
+        member.access_level = access_level
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
         member.save()