qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/gitlab_members.py

@@ -1,11 +1,11 @@
-import enum
 import logging
 from collections.abc import Callable
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
+from gitlab.v4.objects import (
+    Group,
+    GroupMember,
+)
 from pydantic import BaseModel
 
 from reconcile import queries
@@ -26,6 +26,7 @@ from reconcile.gql_definitions.gitlab_members.permissions import (
 )
 from reconcile.utils import gql
 from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.exceptions import AppInterfaceSettingsError
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.pagerduty_api import (
@@ -40,132 +41,93 @@ QONTRACT_INTEGRATION = "gitlab-members"
 
 class GitlabUser(BaseModel):
     user: str
-    access_level: str
+    access_level: int
 
 
-State = dict[str, list[GitlabUser]]
+class CurrentStateSpec(BaseModel):
+    members: dict[str, GroupMember]
 
+    class Config:
+        arbitrary_types_allowed = True
 
-class Action(enum.Enum):
-    add_user_to_group = enum.auto()
-    remove_user_from_group = enum.auto()
-    change_access = enum.auto()
 
+class DesiredStateSpec(BaseModel):
+    members: dict[str, GitlabUser]
 
-class Diff(BaseModel):
-    action: Action
-    group: str
-    user: str
-    access_level: str
+    class Config:
+        arbitrary_types_allowed = True
 
 
-def get_current_state(instance: GitlabInstanceV1, gl: GitLabApi) -> State:
+CurrentState = dict[str, CurrentStateSpec]
+DesiredState = dict[str, DesiredStateSpec]
+
+
+def get_current_state(
+    instance: GitlabInstanceV1, gl: GitLabApi, gitlab_groups_map: dict[str, Group]
+) -> CurrentState:
     """Get current gitlab group members for all managed groups."""
     return {
-        g: [
-            GitlabUser(user=u["user"], access_level=u["access_level"])
-            for u in gl.get_group_members(g)
-        ]
+        g: CurrentStateSpec(
+            members={
+                u.username: u for u in gl.get_group_members(gitlab_groups_map.get(g))
+            },
+        )
         for g in instance.managed_groups
     }
 
 
+def add_or_update_user(
+    desired_state_spec: DesiredStateSpec, gitlab_user: GitlabUser
+) -> None:
+    existing_user = desired_state_spec.members.get(gitlab_user.user)
+    if not existing_user:
+        desired_state_spec.members[gitlab_user.user] = gitlab_user
+    else:
+        existing_user.access_level = max(
+            existing_user.access_level, gitlab_user.access_level
+        )
+
+
 def get_desired_state(
     instance: GitlabInstanceV1,
     pagerduty_map: PagerDutyMap,
     permissions: list[PermissionGitlabGroupMembershipV1],
     all_users: list[User],
-) -> State:
+) -> DesiredState:
     """Fetch all desired gitlab users from app-interface."""
-    desired_group_members: State = {g: [] for g in instance.managed_groups}
-    for g in desired_group_members:
-        for p in permissions:
-            if p.group == g:
-                for r in p.roles or []:
-                    for u in (r.users or []) + (r.bots or []):
-                        desired_group_members[g].append(
-                            GitlabUser(user=u.org_username, access_level=p.access)
-                        )
-                if p.pagerduty:
-                    usernames_from_pagerduty = get_usernames_from_pagerduty(
-                        p.pagerduty,
-                        all_users,
-                        g,
-                        pagerduty_map,
-                        get_username_method=lambda u: u.org_username,
-                    )
-                    for u in usernames_from_pagerduty:
-                        desired_group_members[g].append(
-                            GitlabUser(user=u, access_level=p.access)
-                        )
-
+    desired_group_members: DesiredState = {
+        g: build_desired_state_spec(g, permissions, pagerduty_map, all_users)
+        for g in instance.managed_groups
+    }
     return desired_group_members
 
 
-def calculate_diff(current_state: State, desired_state: State) -> list[Diff]:
-    """Compare current and desired state and return all differences."""
-    diff: list[Diff] = []
-    diff += subtract_states(desired_state, current_state, Action.add_user_to_group)
-    diff += subtract_states(current_state, desired_state, Action.remove_user_from_group)
-    diff += check_access(current_state, desired_state)
-    return diff
-
-
-def subtract_states(
-    from_state: State, subtract_state: State, action: Action
-) -> list[Diff]:
-    """Return diff objects for items in from_state but not in subtract_state."""
-    result = []
-    for f_group, f_users in from_state.items():
-        s_group = subtract_state[f_group]
-        for f_user in f_users:
-            found = False
-            for s_user in s_group:
-                if f_user.user != s_user.user:
-                    continue
-                found = True
-                break
-            if not found:
-                result.append(
-                    Diff(
-                        action=action,
-                        group=f_group,
-                        user=f_user.user,
-                        access_level=f_user.access_level,
-                    )
+def build_desired_state_spec(
+    group_name: str,
+    permissions: list[PermissionGitlabGroupMembershipV1],
+    pagerduty_map: PagerDutyMap,
+    all_users: list[User],
+) -> DesiredStateSpec:
+    desired_state_spec = DesiredStateSpec(members={})
+    for p in permissions:
+        if p.group == group_name:
+            p_access_level = GitLabApi.get_access_level(p.access)
+            for r in p.roles or []:
+                for u in (r.users or []) + (r.bots or []):
+                    gu = GitlabUser(user=u.org_username, access_level=p_access_level)
+                    add_or_update_user(desired_state_spec, gu)
+            if p.pagerduty:
+                usernames_from_pagerduty = get_usernames_from_pagerduty(
+                    p.pagerduty,
+                    all_users,
+                    group_name,
+                    pagerduty_map,
+                    get_username_method=lambda u: u.org_username,
                 )
-    return result
-
-
-def check_access(current_state: State, desired_state: State) -> list[Diff]:
-    """Return diff objects for item where access level is different."""
-    result = []
-    for d_group, d_users in desired_state.items():
-        c_group = current_state[d_group]
-        for d_user in d_users:
-            for c_user in c_group:
-                if d_user.user == c_user.user:
-                    if d_user.access_level != c_user.access_level:
-                        result.append(
-                            Diff(
-                                action=Action.change_access,
-                                group=d_group,
-                                user=c_user.user,
-                                access_level=d_user.access_level,
-                            )
-                        )
-                    break
-    return result
-
-
-def act(diff: Diff, gl: GitLabApi) -> None:
-    """Apply a diff object."""
-    if diff.action == Action.remove_user_from_group:
-        gl.remove_group_member(diff.group, diff.user)
-    if diff.action == Action.add_user_to_group:
-        gl.add_group_member(diff.group, diff.user, diff.access_level)
-    if diff.action == Action.change_access:
-        gl.change_access(diff.group, diff.user, diff.access_level)
+                for u in usernames_from_pagerduty:
+                    gu = GitlabUser(user=u, access_level=p_access_level)
+                    add_or_update_user(desired_state_spec, gu)
+    return desired_state_spec
 
 
 def get_permissions(query_func: Callable) -> list[PermissionGitlabGroupMembershipV1]:
@@ -186,10 +148,15 @@ def get_gitlab_instance(query_func: Callable) -> GitlabInstanceV1:
     raise AppInterfaceSettingsError("No gitlab instance found!")
 
 
+def get_managed_groups_map(group_names: list[str], gl: GitLabApi) -> dict[str, Group]:
+    gitlab_groups = {group_name: gl.get_group(group_name) for group_name in group_names}
+    return gitlab_groups
+
+
 @defer
 def run(
     dry_run: bool,
-    defer: Optional[Callable] = None,
+    defer: Callable | None = None,
 ) -> None:
     gqlapi = gql.get_api()
     # queries
@@ -199,8 +166,6 @@ def run(
     pagerduty_instances = pagerduty_instances_query(
         query_func=gqlapi.query
     ).pagerduty_instances
-    if not pagerduty_instances:
-        raise AppInterfaceSettingsError("no pagerduty instance(s) configured")
 
     # APIs
     secret_reader = SecretReader(queries.get_secret_reader_settings())
@@ -219,17 +184,58 @@ def run(
     pagerduty_map = get_pagerduty_map(
         secret_reader, pagerduty_instances=pagerduty_instances
     )
-
-    # act
-    current_state = get_current_state(instance, gl)
+    managed_groups_map = get_managed_groups_map(instance.managed_groups, gl)
+    current_state = get_current_state(instance, gl, managed_groups_map)
     desired_state = get_desired_state(instance, pagerduty_map, permissions, all_users)
-    diffs = calculate_diff(current_state, desired_state)
-
-    for diff in diffs:
-        logging.info(diff)
-
-        if not dry_run:
-            act(diff, gl)
+    for group_name in instance.managed_groups:
+        reconcile_gitlab_members(
+            current_state_spec=current_state.get(group_name),
+            desired_state_spec=desired_state.get(group_name),
+            group=managed_groups_map.get(group_name),
+            gl=gl,
+            dry_run=dry_run,
+        )
+
+
+def reconcile_gitlab_members(
+    current_state_spec: CurrentStateSpec | None,
+    desired_state_spec: DesiredStateSpec | None,
+    group: Group | None,
+    gl: GitLabApi,
+    dry_run: bool,
+) -> None:
+    if current_state_spec and desired_state_spec and group:
+        diff_data = diff_mappings(
+            current=current_state_spec.members,
+            desired=desired_state_spec.members,
+            equal=lambda current, desired: current.access_level == desired.access_level,
+        )
+        for key, gitlab_user in diff_data.add.items():
+            logging.info([
+                key,
+                "add_user_to_group",
+                group.name,
+                gl.get_access_level_string(gitlab_user.access_level),
+            ])
+            if not dry_run:
+                gl.add_group_member(group, gitlab_user)
+        for key, group_member in diff_data.delete.items():
+            logging.info([
+                key,
+                "remove_user_from_group",
+                group.name,
+            ])
+            if not dry_run:
+                gl.remove_group_member(group, group_member.id)
+        for key, diff_pair in diff_data.change.items():
+            logging.info([
+                key,
+                "change_access",
+                group.name,
+                gl.get_access_level_string(diff_pair.desired.access_level),
+            ])
+            if not dry_run:
+                gl.change_access(diff_pair.current, diff_pair.desired.access_level)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:

reconcile/gitlab_owners.py

@@ -31,18 +31,19 @@ class MRApproval:
     between the approval messages the the project owners.
     """
 
-    def __init__(self, gitlab_client, merge_request, owners, dry_run):
+    def __init__(self, gitlab_client, merge_request, owners, dry_run, persistent_lgtm):
         self.gitlab = gitlab_client
         self.mr = merge_request
         self.owners = owners
         self.dry_run = dry_run
+        self.persistent_lgtm = persistent_lgtm
 
+        # Get the date of the most recent commit (top commit) in the MR, but avoid comparing against None
+        self.top_commit_created_at = dateparser.parse("2000-01-01")
         commits = self.mr.commits()
         if commits:
             top_commit = next(commits)
             self.top_commit_created_at = dateparser.parse(top_commit.created_at)
-        else:
-            self.top_commit_created_at = None
 
     def get_change_owners_map(self):
         """
@@ -50,7 +51,7 @@ class MRApproval:
         that change.
         """
         change_owners_map = {}
-        paths = self.gitlab.get_merge_request_changed_paths(self.mr.iid)
+        paths = self.gitlab.get_merge_request_changed_paths(self.mr)
         for path in paths:
             owners = self.owners.get_path_owners(path)
             path_approvers = owners["approvers"]
@@ -75,9 +76,8 @@ class MRApproval:
         Collects the usernames of all the '/lgtm' comments.
         """
         lgtms = []
-        comments = self.gitlab.get_merge_request_comments(self.mr.iid)
+        comments = self.gitlab.get_merge_request_comments(self.mr)
         for comment in comments:
-
             # Only interested in '/lgtm' comments
             if comment["body"] != "/lgtm":
                 continue
@@ -85,7 +85,10 @@ class MRApproval:
             # Only interested in comments created after the top commit
             # creation time
             comment_created_at = dateparser.parse(comment["created_at"])
-            if comment_created_at < self.top_commit_created_at:
+            if (
+                comment_created_at < self.top_commit_created_at
+                and not self.persistent_lgtm
+            ):
                 continue
 
             lgtms.append(comment["username"])
@@ -143,7 +146,7 @@ class MRApproval:
         # Now, since we have a report, let's check if that report was
         # already used for a comment
         formatted_report = self.format_report(report)
-        comments = self.gitlab.get_merge_request_comments(self.mr.iid)
+        comments = self.gitlab.get_merge_request_comments(self.mr)
         for comment in comments:
             # Only interested on our own comments
             if comment["username"] != self.gitlab.user.username:
@@ -160,15 +163,13 @@ class MRApproval:
             comment_created_at = dateparser.parse(comment["created_at"])
             if comment_created_at < self.top_commit_created_at:
                 # Deleting stale comments
-                _LOG.info(
-                    [
-                        f"Project:{self.gitlab.project.id} "
-                        f"Merge Request:{self.mr.iid} "
-                        f"- removing stale comment"
-                    ]
-                )
+                _LOG.info([
+                    f"Project:{self.gitlab.project.id} "
+                    f"Merge Request:{self.mr.iid} "
+                    f"- removing stale comment"
+                ])
                 if not self.dry_run:
-                    self.gitlab.delete_gitlab_comment(self.mr.iid, comment["id"])
+                    self.gitlab.delete_comment(comment["note"])
                 continue
 
             # At this point, we've found an approval comment comment
@@ -186,8 +187,7 @@ class MRApproval:
         return approval_status
 
     def has_approval_label(self):
-        labels = self.gitlab.get_merge_request_labels(self.mr.iid)
-        return APPROVED in labels
+        return APPROVED in self.mr.labels
 
     @staticmethod
     def format_report(report):
@@ -218,7 +218,6 @@ class MRApproval:
                 closest_approvers.append(new_group)
 
         if closest_approvers:
-
             if len(closest_approvers) == 1:
                 markdown_report += (
                     f"{COMMENT_PREFIX} You will need a "
@@ -263,7 +262,6 @@ class MRApproval:
                 continue
 
             for closest_reviewer in owners["closest_reviewers"]:
-
                 there = False
                 for group in closest_approvers:
                     if closest_reviewer in group:
@@ -285,7 +283,6 @@ class MRApproval:
                 continue
 
             for reviewer in owners["reviewers"]:
-
                 there = False
                 for group in closest_approvers:
                     if reviewer in group:
@@ -313,7 +310,7 @@ class MRApproval:
 
 @defer
 def act(repo, dry_run, instance, settings, defer=None):
-    gitlab_cli = GitLabApi(instance, project_url=repo, settings=settings)
+    gitlab_cli = GitLabApi(instance, project_url=repo["url"], settings=settings)
     defer(gitlab_cli.cleanup)
     project_owners = RepoOwners(
         git_cli=gitlab_cli, ref=gitlab_cli.project.default_branch
@@ -325,72 +322,62 @@ def act(repo, dry_run, instance, settings, defer=None):
             merge_request=mr,
             owners=project_owners,
             dry_run=dry_run,
+            persistent_lgtm=repo.get("gitlabRepoOwners", {}).get("persistentLgtm", None)
+            or False,
         )
 
         if mr_approval.top_commit_created_at is None:
-            _LOG.info(
-                [
-                    f"Project:{gitlab_cli.project.id} "
-                    f"Merge Request:{mr.iid} "
-                    f"- skipping"
-                ]
-            )
+            _LOG.info([
+                f"Project:{gitlab_cli.project.id} "
+                f"Merge Request:{mr.iid} "
+                f"- skipping"
+            ])
             continue
 
         approval_status = mr_approval.get_approval_status()
         if approval_status["approved"]:
             if mr_approval.has_approval_label():
-                _LOG.info(
-                    [
-                        f"Project:{gitlab_cli.project.id} "
-                        f"Merge Request:{mr.iid} "
-                        f"- already approved"
-                    ]
-                )
-                continue
-            _LOG.info(
-                [
+                _LOG.info([
                     f"Project:{gitlab_cli.project.id} "
                     f"Merge Request:{mr.iid} "
-                    f"- approving now"
-                ]
-            )
+                    f"- already approved"
+                ])
+                continue
+            _LOG.info([
+                f"Project:{gitlab_cli.project.id} "
+                f"Merge Request:{mr.iid} "
+                f"- approving now"
+            ])
             if not dry_run:
-                gitlab_cli.add_label_to_merge_request(mr.iid, APPROVED)
+                gitlab_cli.add_label_to_merge_request(mr, APPROVED)
             continue
 
         if not dry_run:
             if mr_approval.has_approval_label():
-                _LOG.info(
-                    [
-                        f"Project:{gitlab_cli.project.id} "
-                        f"Merge Request:{mr.iid} "
-                        f"- removing approval"
-                    ]
-                )
-                gitlab_cli.remove_label_from_merge_request(mr.iid, APPROVED)
-
-        if approval_status["report"] is not None:
-            _LOG.info(
-                [
+                _LOG.info([
                     f"Project:{gitlab_cli.project.id} "
                     f"Merge Request:{mr.iid} "
-                    f"- publishing approval report"
-                ]
-            )
+                    f"- removing approval"
+                ])
+                gitlab_cli.remove_label(mr, APPROVED)
+
+        if approval_status["report"] is not None:
+            _LOG.info([
+                f"Project:{gitlab_cli.project.id} "
+                f"Merge Request:{mr.iid} "
+                f"- publishing approval report"
+            ])
 
             if not dry_run:
-                gitlab_cli.remove_label_from_merge_request(mr.iid, APPROVED)
+                gitlab_cli.remove_label(mr, APPROVED)
                 mr.notes.create({"body": approval_status["report"]})
             continue
 
-        _LOG.info(
-            [
-                f"Project:{gitlab_cli.project.id} "
-                f"Merge Request:{mr.iid} "
-                f"- not fully approved"
-            ]
-        )
+        _LOG.info([
+            f"Project:{gitlab_cli.project.id} "
+            f"Merge Request:{mr.iid} "
+            f"- not fully approved"
+        ])
 
 
 def run(dry_run, thread_pool_size=10):