qontract-reconcile 0.10.1rc1029__py3-none-any.whl → 0.10.1rc1030__py3-none-any.whl

{qontract_reconcile-0.10.1rc1029.dist-info → qontract_reconcile-0.10.1rc1030.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1029
+Version: 0.10.1rc1030
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1029.dist-info → qontract_reconcile-0.10.1rc1030.dist-info}/RECORD

@@ -35,7 +35,7 @@ reconcile/gitlab_labeler.py,sha256=4xJHmVX155fclrHqkR926sL1GH6RTN5XfZ8PnqNXbRA,4
 reconcile/gitlab_members.py,sha256=PrJE9OhDRdGG_gHM_77nQojLb4B18jtUu8DxgLsRS88,8417
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
 reconcile/gitlab_owners.py,sha256=sn9njaKOtqcvnhi2qtm-faAfAR4zNqflbSuusA9RUuI,13456
-reconcile/gitlab_permissions.py,sha256=wq0jbWWPN5iOWnRLuZeemCgKh4a_iGy2d4Iq3WFJykM,3636
+reconcile/gitlab_permissions.py,sha256=tEjbpr-QH03vZ4L6CuVJhkgA3oaB_gyJ7d58MTR4JJQ,7389
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=gvOhVklJDeMPURxLjV30Q4hnLET3BZ-NeEEtQBoo_E0,9500
 reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
@@ -518,7 +518,7 @@ reconcile/test/test_github_repo_invites.py,sha256=WrPn5ROAoJYK0ihzlZcFR0V9Pu2HcM
 reconcile/test/test_gitlab_housekeeping.py,sha256=ScD9Tgf9OOgGfAFfTy6Kn2222l2wH_A3gMRKdpvoWe0,10053
 reconcile/test/test_gitlab_labeler.py,sha256=PmYXiU2g0_O5OTdMGPzdeqBAfat92U9bhjjfeyiGSmQ,4336
 reconcile/test/test_gitlab_members.py,sha256=yjfQRUFG_F0kLYegax4_ec5VIBAnCPrvAgqMcN1GXzc,9985
-reconcile/test/test_gitlab_permissions.py,sha256=vPFEsdjyP-lO8pc2rN6acMns3Sjz9YJs16msbBR8DZc,2736
+reconcile/test/test_gitlab_permissions.py,sha256=aMf5SUeVp-aQ1bWGQPQLYa85auzRlyfZVTWqyybJ6mo,5850
 reconcile/test/test_instrumented_wrappers.py,sha256=CZzhnQH0c4i7-Rxjg7-0dfFMvVPegLHL46z5NHOOCwo,608
 reconcile/test/test_integrations_manager.py,sha256=xpyQAVz57wAbovrcQzAeuyq8VzdItUyW2d2kp1WW_5c,38184
 reconcile/test/test_jenkins_worker_fleets.py,sha256=o1jlT7OBBSgu0M3iI4xMdz_x6SciF7yhNBpLk5gTJfg,2361
@@ -674,7 +674,7 @@ reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,
 reconcile/utils/git.py,sha256=JkpbUO10oBTtNHZ1IhjyG6dTOUizc7I5H0vm7NvDVNw,1409
 reconcile/utils/git_secrets.py,sha256=y1rEhwA8DyDpBSAEuhMS7Y2X3mpxT2zQ4zyDFkhLe_g,1936
 reconcile/utils/github_api.py,sha256=R8OvqyPdnRqvP-Efnv9RvIcbBlb4M0KC4RlbnJMD0Tg,2426
-reconcile/utils/gitlab_api.py,sha256=WfhLhOp3-cOwgptiCNPhWja74Lo41ZIlJ6HFSWaIDRw,30512
+reconcile/utils/gitlab_api.py,sha256=XCBME1p42D6dbedbcLlFt1VmTLZLlOjZEnusoXEr4ps,29029
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=NqxosFCInLy_dF_fcXnRYcZgsThedSyv_I3U8k2SLMg,14161
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -865,8 +865,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1029.dist-info/METADATA,sha256=w9MZTHnW2xZ0UpGosbgS2pYbE8FSj9XvFbw_uJnNjRw,2213
-qontract_reconcile-0.10.1rc1029.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1029.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1029.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1029.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1030.dist-info/METADATA,sha256=1Kn_s1Qs4zQwmniZl-bI8OXCfwOCb0q2uUMwePoypzk,2213
+qontract_reconcile-0.10.1rc1030.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1030.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1030.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1030.dist-info/RECORD,,

reconcile/gitlab_permissions.py

@@ -1,21 +1,146 @@
 import itertools
 import logging
+from dataclasses import dataclass
 from typing import Any
 
-from gitlab.const import MAINTAINER_ACCESS
+from gitlab.v4.objects import (
+    GroupProject,
+    Project,
+)
 from sretoolbox.utils import threaded
 
 from reconcile import queries
 from reconcile.utils import batches
 from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.unleash import get_feature_toggle_state
 
 QONTRACT_INTEGRATION = "gitlab-permissions"
 APP_SRE_GROUP_NAME = "app-sre"
+GROUP_ACCESS = "maintainer"
 PAGE_SIZE = 100
 
 
+@dataclass
+class GroupSpec:
+    group_name: str
+    group_access_level: int
+
+
+class GroupAccessLevelError(Exception):
+    pass
+
+
+class GroupPermissionHandler:
+    def __init__(
+        self, gl: GitLabApi, group_name: str, access: str, dry_run: bool
+    ) -> None:
+        self.gl = gl
+        self.dry_run = dry_run
+        self.access_level_string = access
+        self.access_level = self.gl.get_access_level(access)
+        self.group = self.gl.get_group(group_name)
+
+    def run(self, repos: list[str]) -> None:
+        # filter projects belonging to the same group and remove it from the state data
+        filtered_project_repos = self.filter_group_owned_projects(repos)
+        desired_state = {
+            project_repo_url: GroupSpec(self.group.name, self.access_level)
+            for project_repo_url in filtered_project_repos
+        }
+        # get all projects shared with group
+        shared_projects = self.gl.get_items(self.group.shared_projects.list)
+        current_state = {
+            project.web_url: self.extract_group_spec(project)
+            for project in shared_projects
+        }
+        self.reconcile(desired_state, current_state)
+
+    def filter_group_owned_projects(self, repos: list[str]) -> set[str]:
+        # get only the projects that are owned by  group and its sub groups
+        query = {"with_shared": False, "include_subgroups": True}
+        group_owned_projects = self.gl.get_items(
+            self.group.projects.list, query_parameters=query
+        )
+        group_owned_repo_list = {project.web_url for project in group_owned_projects}
+        return set(repos) - group_owned_repo_list
+
+    def extract_group_spec(self, project: GroupProject) -> GroupSpec:
+        return next(
+            GroupSpec(
+                group_name=self.group.name,
+                group_access_level=group["group_access_level"],
+            )
+            for group in project.shared_with_groups
+            if group["group_id"] == self.group.id
+        )
+
+    def can_share_project(self, project: Project) -> bool:
+        # check if user have access greater or equal access to be shared with the group
+        user = project.members_all.get(id=self.gl.user.id)
+        return user.access_level >= self.access_level
+
+    def reconcile(
+        self,
+        desired_state: dict[str, GroupSpec],
+        current_state: dict[str, GroupSpec],
+    ) -> None:
+        # get the diff data
+        diff_data = diff_mappings(
+            current=current_state,
+            desired=desired_state,
+            equal=lambda current, desired: current.group_access_level
+            == desired.group_access_level,
+        )
+        errors: list[Exception] = []
+        for repo in diff_data.add:
+            project = self.gl.get_project(repo)
+            if not self.can_share_project(project):
+                errors.append(
+                    GroupAccessLevelError(
+                        f"{repo} is not shared with {self.gl.user.username} as {self.access_level_string}"
+                    )
+                )
+                continue
+            logging.info([
+                "share",
+                repo,
+                self.group.name,
+                self.access_level_string,
+            ])
+            if not self.dry_run:
+                self.gl.share_project_with_group(
+                    project=project,
+                    group_id=self.group.id,
+                    access_level=self.access_level,
+                )
+        for repo in diff_data.change:
+            project = self.gl.get_project(repo)
+            if not self.can_share_project(project):
+                errors.append(
+                    GroupAccessLevelError(
+                        f"{repo} is not shared with {self.gl.user.username} as {self.access_level_string}"
+                    )
+                )
+                continue
+            logging.info([
+                "reshare",
+                repo,
+                self.group.name,
+                self.access_level_string,
+            ])
+            if not self.dry_run:
+                self.gl.share_project_with_group(
+                    project=project,
+                    group_id=self.group.id,
+                    access_level=self.access_level,
+                    reshare=True,
+                )
+        if errors:
+            raise ExceptionGroup("Reconcile errors occurred", errors)
+
+
 def get_members_to_add(repo, gl, app_sre):
     maintainers = get_all_app_sre_maintainers(repo, gl, app_sre)
     if maintainers is None:
@@ -56,7 +181,10 @@ def run(dry_run, thread_pool_size=10, defer=None):
         default=False,
     )
     if share_with_group_enabled:
-        share_project_with_group(gl, repos, dry_run)
+        group_permission_handler = GroupPermissionHandler(
+            gl=gl, group_name=APP_SRE_GROUP_NAME, access=GROUP_ACCESS, dry_run=dry_run
+        )
+        group_permission_handler.run(repos=repos)
     else:
         share_project_with_group_members(gl, repos, thread_pool_size, dry_run)
 
@@ -75,26 +203,6 @@ def share_project_with_group_members(
             gl.add_project_member(m["repo"], m["user"])
 
 
-def share_project_with_group(gl: GitLabApi, repos: list[str], dry_run: bool) -> None:
-    # get repos not owned by app-sre
-    non_app_sre_project_repos = {repo for repo in repos if "/app-sre/" not in repo}
-    group_id, shared_projects = gl.get_group_id_and_shared_projects(APP_SRE_GROUP_NAME)
-    shared_project_repos = shared_projects.keys()
-    repos_to_share = non_app_sre_project_repos - shared_project_repos
-    repos_to_reshare = {
-        repo
-        for repo in non_app_sre_project_repos
-        if (group_data := shared_projects.get(repo))
-        and group_data["group_access_level"] < MAINTAINER_ACCESS
-    }
-    for repo in repos_to_share:
-        gl.share_project_with_group(repo_url=repo, group_id=group_id, dry_run=dry_run)
-    for repo in repos_to_reshare:
-        gl.share_project_with_group(
-            repo_url=repo, group_id=group_id, dry_run=dry_run, reshare=True
-        )
-
-
 def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
     instance = queries.get_gitlab_instance()
     return {

reconcile/test/test_gitlab_permissions.py

@@ -1,7 +1,17 @@
 from unittest.mock import MagicMock, create_autospec
 
 import pytest
-from gitlab.v4.objects import CurrentUser, GroupMember
+from gitlab.v4.objects import (
+    CurrentUser,
+    Group,
+    GroupMember,
+    GroupProjectManager,
+    Project,
+    ProjectMember,
+    ProjectMemberAllManager,
+    SharedProject,
+    SharedProjectManager,
+)
 from pytest_mock import MockerFixture
 
 from reconcile import gitlab_permissions
@@ -23,6 +33,7 @@ def mocked_gl() -> MagicMock:
     gl.server = "test_server"
     gl.user = create_autospec(CurrentUser)
     gl.user.username = "test_name"
+    gl.user.id = 1234
     return gl
 
 
@@ -49,13 +60,25 @@ def test_run_share_with_group(
     mocker.patch(
         "reconcile.gitlab_permissions.get_feature_toggle_state"
     ).return_value = True
-    mocked_gl.get_group_id_and_shared_projects.return_value = (
-        1234,
-        {"https://test.com": {"group_access_level": 30}},
+    group = create_autospec(Group, id=1234)
+    group.name = "app-sre"
+    group.projects = create_autospec(GroupProjectManager)
+    group.shared_projects = create_autospec(SharedProjectManager)
+    mocked_gl.get_items.side_effect = [
+        [],
+        [],
+    ]
+    mocked_gl.get_group.return_value = group
+    mocked_gl.get_access_level.return_value = 40
+    project = create_autospec(Project, web_url="https://test.com")
+    project.members_all = create_autospec(ProjectMemberAllManager)
+    project.members_all.get.return_value = create_autospec(
+        ProjectMember, id=mocked_gl.user.id, access_level=40
     )
+    mocked_gl.get_project.return_value = project
     gitlab_permissions.run(False, thread_pool_size=1)
     mocked_gl.share_project_with_group.assert_called_once_with(
-        repo_url="https://test-gitlab.com", group_id=1234, dry_run=False
+        project, group_id=1234, access_level=40
     )
 
 
@@ -66,11 +89,76 @@ def test_run_reshare_with_group(
     mocker.patch(
         "reconcile.gitlab_permissions.get_feature_toggle_state"
     ).return_value = True
-    mocked_gl.get_group_id_and_shared_projects.return_value = (
-        1234,
-        {"https://test-gitlab.com": {"group_access_level": 30}},
+    group = create_autospec(Group, id=1234)
+    group.name = "app-sre"
+    group.projects = create_autospec(GroupProjectManager)
+    group.shared_projects = create_autospec(SharedProjectManager)
+    mocked_gl.get_items.side_effect = [
+        [],
+        [
+            create_autospec(
+                SharedProject,
+                web_url="https://test-gitlab.com",
+                shared_with_groups=[
+                    {
+                        "group_access_level": 30,
+                        "group_name": "app-sre",
+                        "group_id": 1234,
+                    }
+                ],
+            )
+        ],
+    ]
+    mocked_gl.get_group.return_value = group
+    mocked_gl.get_access_level.return_value = 40
+    project = create_autospec(Project, web_url="https://test-gitlab.com")
+    project.members_all = create_autospec(ProjectMemberAllManager)
+    project.members_all.get.return_value = create_autospec(
+        ProjectMember, id=mocked_gl.user.id, access_level=40
     )
+    mocked_gl.get_project.return_value = project
     gitlab_permissions.run(False, thread_pool_size=1)
     mocked_gl.share_project_with_group.assert_called_once_with(
-        repo_url="https://test-gitlab.com", group_id=1234, dry_run=False, reshare=True
+        project=project, group_id=1234, access_level=40, reshare=True
+    )
+
+
+def test_run_share_with_group_failed(
+    mocked_queries: MagicMock, mocker: MockerFixture, mocked_gl: MagicMock
+) -> None:
+    mocker.patch("reconcile.gitlab_permissions.GitLabApi").return_value = mocked_gl
+    mocker.patch(
+        "reconcile.gitlab_permissions.get_feature_toggle_state"
+    ).return_value = True
+    group = create_autospec(Group, id=1234)
+    group.name = "app-sre"
+    group.projects = create_autospec(GroupProjectManager)
+    group.shared_projects = create_autospec(SharedProjectManager)
+    group.projects = create_autospec(GroupProjectManager)
+    group.shared_projects = create_autospec(SharedProjectManager)
+    mocked_gl.get_items.side_effect = [
+        [],
+        [
+            create_autospec(
+                SharedProject,
+                web_url="https://test-gitlab.com",
+                shared_with_groups=[
+                    {
+                        "group_access_level": 30,
+                        "group_name": "app-sre",
+                        "group_id": 134,
+                    }
+                ],
+            )
+        ],
+    ]
+    mocked_gl.get_group.return_value = group
+    mocked_gl.get_access_level.return_value = 40
+    project = create_autospec(Project)
+    project.members_all = create_autospec(ProjectMemberAllManager)
+    project.members_all.get.return_value = create_autospec(
+        ProjectMember, id=mocked_gl.user.id, access_level=10
     )
+    mocked_gl.get_project.return_value = project
+    with pytest.raises(Exception):
+        gitlab_permissions.run(False, thread_pool_size=1)

reconcile/utils/gitlab_api.py

@@ -262,51 +262,16 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
 
     def share_project_with_group(
         self,
-        repo_url: str,
+        project: Project,
         group_id: int,
-        dry_run: bool,
-        access: str = "maintainer",
+        access_level: int,
         reshare: bool = False,
     ) -> None:
-        project = self.get_project(repo_url)
-        if project is None:
-            return None
-        access_level = self.get_access_level(access)
-        # check if we have 'access_level' access so we can  add the group with same role.
-        members = self.get_items(
-            project.members_all.list, query_parameters={"user_ids": self.user.id}
-        )
-        if not any(
-            self.user.id == member.id and member.access_level >= access_level
-            for member in members
-        ):
-            logging.error(
-                "%s is not shared with %s as %s",
-                repo_url,
-                self.user.username,
-                access,
-            )
-            return None
-        logging.info(["add_group_as_maintainer", repo_url, group_id])
-        if not dry_run:
-            if reshare:
-                gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-                project.unshare(group_id)
+        if reshare:
             gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-            project.share(group_id, access_level)
-
-    def get_group_id_and_shared_projects(
-        self, group_name: str
-    ) -> tuple[int, dict[str, Any]]:
+            project.unshare(group_id)
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        group = self.gl.groups.get(group_name)
-        shared_projects = self.get_items(group.projects.list)
-        return group.id, {
-            project.web_url: shared_group
-            for project in shared_projects
-            for shared_group in project.shared_with_groups
-            if shared_group["group_id"] == group.id
-        }
+        project.share(group_id, access_level)
 
     @staticmethod
     def _is_bot_username(username: str) -> bool:
@@ -414,10 +379,9 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
         if access == "guest":
             return GUEST_ACCESS
 
-    def get_group_id_and_projects(self, group_name: str) -> tuple[str, list[str]]:
+    def get_group(self, group_name: str) -> Group:
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
-        group = self.gl.groups.get(group_name)
-        return group.id, [p.name for p in self.get_items(group.projects.list)]
+        return self.gl.groups.get(group_name)
 
     def create_project(self, group_id, project):
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()