qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/gitlab_permissions.py

@@ -1,24 +1,156 @@
 import itertools
 import logging
+from dataclasses import dataclass
 from typing import Any
 
+from gitlab.exceptions import GitlabGetError
+from gitlab.v4.objects import (
+    GroupProject,
+    Project,
+)
 from sretoolbox.utils import threaded
 
 from reconcile import queries
+from reconcile.utils import batches
 from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.unleash import get_feature_toggle_state
 
 QONTRACT_INTEGRATION = "gitlab-permissions"
+APP_SRE_GROUP_NAME = "app-sre"
+GROUP_ACCESS = "maintainer"
+PAGE_SIZE = 100
+
+
+@dataclass
+class GroupSpec:
+    group_name: str
+    group_access_level: int
+
+
+class GroupAccessLevelError(Exception):
+    pass
+
+
+class GroupPermissionHandler:
+    def __init__(
+        self, gl: GitLabApi, group_name: str, access: str, dry_run: bool
+    ) -> None:
+        self.gl = gl
+        self.dry_run = dry_run
+        self.access_level_string = access
+        self.access_level = self.gl.get_access_level(access)
+        self.group = self.gl.get_group(group_name)
+
+    def run(self, repos: list[str]) -> None:
+        # filter projects belonging to the same group and remove it from the state data
+        filtered_project_repos = self.filter_group_owned_projects(repos)
+        desired_state = {
+            project_repo_url: GroupSpec(self.group.name, self.access_level)
+            for project_repo_url in filtered_project_repos
+        }
+        # get all projects shared with group
+        shared_projects = self.gl.get_items(self.group.shared_projects.list)
+        current_state = {
+            project.web_url: self.extract_group_spec(project)
+            for project in shared_projects
+        }
+        self.reconcile(desired_state, current_state)
+
+    def filter_group_owned_projects(self, repos: list[str]) -> set[str]:
+        # get only the projects that are owned by  group and its sub groups
+        query = {"with_shared": False, "include_subgroups": True}
+        group_owned_projects = self.gl.get_items(
+            self.group.projects.list, query_parameters=query
+        )
+        group_owned_repo_list = {project.web_url for project in group_owned_projects}
+        return set(repos) - group_owned_repo_list
+
+    def extract_group_spec(self, project: GroupProject) -> GroupSpec:
+        return next(
+            GroupSpec(
+                group_name=self.group.name,
+                group_access_level=group["group_access_level"],
+            )
+            for group in project.shared_with_groups
+            if group["group_id"] == self.group.id
+        )
+
+    def can_share_project(self, project: Project) -> bool:
+        # check if user have access greater or equal access to be shared with the group
+        try:
+            user = project.members_all.get(id=self.gl.user.id)
+        except GitlabGetError:
+            return False
+        return user.access_level >= self.access_level
+
+    def reconcile(
+        self,
+        desired_state: dict[str, GroupSpec],
+        current_state: dict[str, GroupSpec],
+    ) -> None:
+        # get the diff data
+        diff_data = diff_mappings(
+            current=current_state,
+            desired=desired_state,
+            equal=lambda current, desired: current.group_access_level
+            == desired.group_access_level,
+        )
+        errors: list[Exception] = []
+        for repo in diff_data.add:
+            project = self.gl.get_project(repo)
+            if not self.can_share_project(project):
+                errors.append(
+                    GroupAccessLevelError(
+                        f"{repo} is not shared with {self.gl.user.username} as {self.access_level_string}"
+                    )
+                )
+                continue
+            logging.info([
+                "share",
+                repo,
+                self.group.name,
+                self.access_level_string,
+            ])
+            if not self.dry_run:
+                self.gl.share_project_with_group(
+                    project=project,
+                    group_id=self.group.id,
+                    access_level=self.access_level,
+                )
+        for repo in diff_data.change:
+            project = self.gl.get_project(repo)
+            if not self.can_share_project(project):
+                errors.append(
+                    GroupAccessLevelError(
+                        f"{repo} is not shared with {self.gl.user.username} as {self.access_level_string}"
+                    )
+                )
+                continue
+            logging.info([
+                "reshare",
+                repo,
+                self.group.name,
+                self.access_level_string,
+            ])
+            if not self.dry_run:
+                self.gl.share_project_with_group(
+                    project=project,
+                    group_id=self.group.id,
+                    access_level=self.access_level,
+                    reshare=True,
+                )
+        if errors:
+            raise ExceptionGroup("Reconcile errors occurred", errors)
 
 
 def get_members_to_add(repo, gl, app_sre):
-    maintainers = gl.get_project_maintainers(repo)
+    maintainers = get_all_app_sre_maintainers(repo, gl, app_sre)
     if maintainers is None:
         return []
     if gl.user.username not in maintainers:
-        logging.error(
-            "'{}' is not shared with {} as 'Maintainer'".format(repo, gl.user.username)
-        )
+        logging.error(f"'{repo}' is not shared with {gl.user.username} as 'Maintainer'")
         return []
     members_to_add = [
         {"user": u, "repo": repo} for u in app_sre if u.username not in maintainers
@@ -26,6 +158,20 @@ def get_members_to_add(repo, gl, app_sre):
     return members_to_add
 
 
+def get_all_app_sre_maintainers(repo, gl, app_sre):
+    app_sre_user_ids = [user.id for user in app_sre]
+    chunks = batches.batched(app_sre_user_ids, PAGE_SIZE)
+    app_sre_maintainers = (
+        gl.get_project_maintainers(repo, query=create_user_ids_query(chunk))
+        for chunk in chunks
+    )
+    return list(itertools.chain.from_iterable(app_sre_maintainers))
+
+
+def create_user_ids_query(ids):
+    return {"user_ids": ",".join(str(id) for id in ids)}
+
+
 @defer
 def run(dry_run, thread_pool_size=10, defer=None):
     instance = queries.get_gitlab_instance()
@@ -33,12 +179,27 @@ def run(dry_run, thread_pool_size=10, defer=None):
     gl = GitLabApi(instance, settings=settings)
     if defer:
         defer(gl.cleanup)
-    repos = queries.get_repos(server=gl.server)
+    repos = queries.get_repos(server=gl.server, exclude_manage_permissions=True)
+    share_with_group_enabled = get_feature_toggle_state(
+        "gitlab-permissions-share-with-group",
+        default=False,
+    )
+    if share_with_group_enabled:
+        group_permission_handler = GroupPermissionHandler(
+            gl=gl, group_name=APP_SRE_GROUP_NAME, access=GROUP_ACCESS, dry_run=dry_run
+        )
+        group_permission_handler.run(repos=repos)
+    else:
+        share_project_with_group_members(gl, repos, thread_pool_size, dry_run)
+
+
+def share_project_with_group_members(
+    gl: GitLabApi, repos: list[str], thread_pool_size: int, dry_run: bool
+) -> None:
     app_sre = gl.get_app_sre_group_users()
     results = threaded.run(
         get_members_to_add, repos, thread_pool_size, gl=gl, app_sre=app_sre
     )
-
     members_to_add = list(itertools.chain.from_iterable(results))
     for m in members_to_add:
         logging.info(["add_maintainer", m["repo"], m["user"].username])

reconcile/glitchtip/README.md

@@ -0,0 +1,150 @@
+<font size=24> GlitchTip </font>
+---
+
+[toc]
+
+# Introduction
+
+This integration manages the following glitchtip objects:
+
+* organizations
+* teams
+* projects
+* and users
+
+
+# Development
+
+Development and testing were done for `glitchtip/glitchtip:v2.0.5`
+
+glitchtip `docker-compose.yml` for local development:
+
+```yaml
+x-environment:
+  &default-environment
+  DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres
+  SECRET_KEY: whatever
+  DEBUG: "true"
+  EMAIL_BACKEND: "django.core.mail.backends.console.EmailBackend"
+  PORT: 8000
+  GLITCHTIP_DOMAIN: http://localhost:8000
+  CELERY_WORKER_CONCURRENCY: 1
+x-depends_on:
+  &default-depends_on
+  - postgres
+  - redis
+
+services:
+  postgres:
+    image: postgres:13
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+    restart: unless-stopped
+    volumes:
+      - pg-data:/var/lib/postgresql/data
+    networks:
+      - qontract-development
+  redis:
+    image: redis
+    restart: unless-stopped
+    networks:
+      - qontract-development
+  glitchtip:
+    image: glitchtip/glitchtip:v2.0.5
+    depends_on: *default-depends_on
+    ports:
+      - "8000:8000"
+    environment: *default-environment
+    restart: unless-stopped
+    networks:
+      - qontract-development
+  worker:
+    image: glitchtip/glitchtip:v2.0.5
+    command: ./bin/run-celery-with-beat.sh
+    depends_on: *default-depends_on
+    environment: *default-environment
+    restart: unless-stopped
+    networks:
+      - qontract-development
+  migrate:
+    image: glitchtip/glitchtip:v2.0.5
+    depends_on: *default-depends_on
+    command: "./manage.py migrate"
+    environment: *default-environment
+    networks:
+      - qontract-development
+
+volumes:
+  pg-data:
+
+
+networks:
+  qontract-development:
+    external:
+      name: qontract-development
+```
+
+# Fixtures
+
+The unit tests use this structure and are based on these [app-interface-dev-data](https://gitlab.cee.redhat.com/app-sre/app-interface-dev-data/-/merge_requests/19) definitions.
+
+```mermaid
+classDiagram
+    direction TB
+    %% Instance & Orgs
+    GlitchTipDev_Instance <-- ESA_Organization
+    GlitchTipDev_Instance <-- NASA_Organization
+
+    %% Projects
+    NASA_Organization <-- apollo_11_flight_control_Project
+    NASA_Organization <-- apollo_11_spacecraft_Project
+    ESA_Organization <-- rosetta_flight_control_Project
+    ESA_Organization <-- rosetta_spacecraft_Project
+
+    %% Teams
+    apollo_11_flight_control_Project <-- nasa_flight_control_Team
+    apollo_11_flight_control_Project <-- nasa_pilots_Team
+    apollo_11_spacecraft_Project <-- nasa_pilots_Team
+    rosetta_flight_control_Project <-- esa_flight_control_Team
+    rosetta_flight_control_Project <-- esa_pilots_Team
+    rosetta_spacecraft_Project <-- esa_pilots_Team
+
+    %% Users
+    %% pilot
+    ESA_Organization <-- SamanthaCristoforetti_User
+    %% flight-control
+    ESA_Organization <-- MatthiasMaurer_User
+    ESA_Organization <-- TimPeake_User
+    %% owner role in organizations;
+    ESA_Organization <-- GlobalFlightDirector_User
+    %% automation account, must be ignored
+    ESA_Organization <-- sd_app_sre_glitchtip_User
+
+    %% automation account, must be ignored
+    NASA_Organization <-- sd_app_sre_glitchtip_User
+    %% owner role in organizations; invited to nasa-flight-control but not accepted yet
+    NASA_Organization <-- GlobalFlightDirector_User
+    %% pilots
+    NASA_Organization <-- NeilArmstrong_User
+    %% user account not created yet, invited but user ignored the invitation
+    NASA_Organization <-- BuzzAldrin_User
+    %% ordinary user
+    NASA_Organization <-- MichaelCollins_User
+
+    %% Team memberships
+    esa_pilots_Team <-- SamanthaCristoforetti_User
+    esa_flight_control_Team <-- MatthiasMaurer_User
+    esa_flight_control_Team <-- TimPeake_User
+    esa_flight_control_Team <-- GlobalFlightDirector_User
+
+    nasa_flight_control_Team <-- GlobalFlightDirector_User
+    nasa_pilots_Team <-- NeilArmstrong_User
+    nasa_pilots_Team <-- BuzzAldrin_User
+    nasa_flight_control_Team <-- MichaelCollins_User
+```
+
+# Links
+
+* [Design Doc](https://gitlab.cee.redhat.com/service/app-interface/-/blob/d12d7faa9d6136da69e4113ccbbed54781319173/docs/app-sre/design-docs/glitchtip.md)
+* [GlitchTip App](https://visual-app-interface.devshift.net/services#/services/glitchtip/app.yml)
+* [GlitchTip Stage Instance](https://glitchtip.stage.devshift.net/login)

reconcile/glitchtip/integration.py

@@ -1,13 +1,10 @@
 from collections.abc import (
+    Callable,
     Iterable,
     Sequence,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
-from reconcile import queries
 from reconcile.glitchtip.reconciler import GlitchtipReconciler
 from reconcile.gql_definitions.glitchtip.glitchtip_instance import (
     DEFINITION as GLITCHTIP_INSTANCE_DEFINITION,
@@ -19,16 +16,18 @@ from reconcile.gql_definitions.glitchtip.glitchtip_project import (
     DEFINITION as GLITCHTIP_PROJECT_DEFINITION,
 )
 from reconcile.gql_definitions.glitchtip.glitchtip_project import (
-    GlitchtipProjectsV1,
+    GlitchtipProjectV1,
     RoleV1,
 )
 from reconcile.gql_definitions.glitchtip.glitchtip_project import (
     query as glitchtip_project_query,
 )
-from reconcile.gql_definitions.glitchtip.glitchtip_settings import (
-    query as glitchtip_settings_query,
+from reconcile.ldap_groups.integration import LdapGroupsIntegration
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
 )
 from reconcile.utils import gql
+from reconcile.utils.defer import defer
 from reconcile.utils.glitchtip import (
     GlitchtipClient,
     Organization,
@@ -36,9 +35,15 @@ from reconcile.utils.glitchtip import (
     Team,
     User,
 )
-from reconcile.utils.secret_reader import SecretReader
+from reconcile.utils.internal_groups.client import InternalGroupsClient
+from reconcile.utils.rest_api_base import BearerTokenAuth
+from reconcile.utils.secret_reader import (
+    SecretReaderBase,
+    create_secret_reader,
+)
 
 QONTRACT_INTEGRATION = "glitchtip"
+DEFAULT_MEMBER_ROLE = "member"
 
 
 def filter_users(users: Iterable[User], ignore_users: Iterable[str]) -> list[User]:
@@ -50,7 +55,7 @@ def get_user_role(organization: Organization, roles: RoleV1) -> str:
         if role.organization.name == organization.name:
             return role.role
     # this can not be reached due to GQL but makes mypy happy
-    return "member"
+    return DEFAULT_MEMBER_ROLE
 
 
 class GlitchtipException(Exception):
@@ -79,7 +84,9 @@ def fetch_current_state(
 
 
 def fetch_desired_state(
-    glitchtip_projects: Sequence[GlitchtipProjectsV1], mail_domain: str
+    glitchtip_projects: Sequence[GlitchtipProjectV1],
+    mail_domain: str,
+    internal_groups_client: InternalGroupsClient,
 ) -> list[Organization]:
     organizations: dict[str, Organization] = {}
     for glitchtip_project in glitchtip_projects:
@@ -88,13 +95,18 @@ def fetch_desired_state(
             Organization(name=glitchtip_project.organization.name),
         )
         project = Project(
-            name=glitchtip_project.name, platform=glitchtip_project.platform
+            name=glitchtip_project.name,
+            platform=glitchtip_project.platform,
+            slug=glitchtip_project.project_id if glitchtip_project.project_id else "",
+            event_throttle_rate=glitchtip_project.event_throttle_rate or 0,
         )
         # Check project is unique within an organization
         if project.name in [p.name for p in organization.projects]:
             raise GlitchtipException(f'project name "{project.name}" already in use!')
         for glitchtip_team in glitchtip_project.teams:
             users: list[User] = []
+
+            # Get users via roles
             for role in glitchtip_team.roles:
                 for role_user in role.users:
                     users.append(
@@ -104,7 +116,19 @@ def fetch_desired_state(
                         )
                     )
 
-            team = Team(name=glitchtip_team.name, users=users)
+            # Get users via ldap
+            for ldap_group in glitchtip_team.ldap_groups or []:
+                for member in internal_groups_client.group(ldap_group).members:
+                    users.append(
+                        User(
+                            email=f"{member.id}@{mail_domain}",
+                            role=glitchtip_team.members_organization_role
+                            or DEFAULT_MEMBER_ROLE,
+                        )
+                    )
+
+            # set(users) will take the first occurrence of a user, so the users from roles will be preferred
+            team = Team(name=glitchtip_team.name, users=set(users))
             project.teams.append(team)
             if team not in organization.teams:
                 organization.teams.append(team)
@@ -116,54 +140,78 @@ def fetch_desired_state(
     return list(organizations.values())
 
 
-def run(dry_run: bool, instance: Optional[str] = None) -> None:
+def get_glitchtip_projects(query_func: Callable) -> list[GlitchtipProjectV1]:
+    glitchtip_projects = (
+        glitchtip_project_query(query_func=query_func).glitchtip_projects or []
+    )
+    for project in glitchtip_projects:
+        # either org.owners or project.app must be set
+        if not project.organization.owners and not project.app:
+            raise ValueError(
+                f"Either owners in organization {project.organization.name} or app must be set for project {project.name}"
+            )
+
+    return glitchtip_projects
+
+
+def get_internal_groups_client(
+    query_func: Callable, secret_reader: SecretReaderBase
+) -> InternalGroupsClient:
+    ldap_groups_settings = LdapGroupsIntegration.get_integration_settings(query_func)
+    secret = secret_reader.read_all_secret(ldap_groups_settings.credentials)
+    return InternalGroupsClient(
+        secret["api_url"],
+        secret["issuer_url"],
+        secret["client_id"],
+        secret["client_secret"],
+    )
+
+
+@defer
+def run(
+    dry_run: bool, instance: str | None = None, defer: Callable | None = None
+) -> None:
     gqlapi = gql.get_api()
-    secret_reader = SecretReader(queries.get_secret_reader_settings())
-    read_timeout = 30
-    max_retries = 3
-    mail_domain = "redhat.com"
-    if _s := glitchtip_settings_query(query_func=gqlapi.query).settings:
-        if _gs := _s[0].glitchtip:
-            if _gs.read_timeout is not None:
-                read_timeout = _gs.read_timeout
-            if _gs.max_retries is not None:
-                max_retries = _gs.max_retries
-            if _gs.mail_domain is not None:
-                mail_domain = _gs.mail_domain
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    internal_groups_client = get_internal_groups_client(gqlapi.query, secret_reader)
+    if defer:
+        defer(internal_groups_client.close)
 
     glitchtip_instances = glitchtip_instance_query(query_func=gqlapi.query).instances
-    glitchtip_projects = (
-        glitchtip_project_query(query_func=gqlapi.query).glitchtip_projects or []
-    )
+    glitchtip_projects = get_glitchtip_projects(query_func=gqlapi.query)
 
     for glitchtip_instance in glitchtip_instances:
         if instance and glitchtip_instance.name != instance:
             continue
 
-        glitchtip_client = GlitchtipClient(
+        with GlitchtipClient(
             host=glitchtip_instance.console_url,
-            token=secret_reader.read_secret(glitchtip_instance.automation_token),
-            read_timeout=read_timeout,
-            max_retries=max_retries,
-        )
-        current_state = fetch_current_state(
-            glitchtip_client=glitchtip_client,
-            # the automation user isn't managed by app-interface (chicken - egg problem), so just ignore it
-            ignore_users=[
-                secret_reader.read_secret(glitchtip_instance.automation_user_email)
-            ],
-        )
-        desired_state = fetch_desired_state(
-            glitchtip_projects=[
-                p
-                for p in glitchtip_projects
-                if p.organization.instance.name == glitchtip_instance.name
-            ],
-            mail_domain=mail_domain,
-        )
+            auth=BearerTokenAuth(
+                secret_reader.read_secret(glitchtip_instance.automation_token)
+            ),
+            read_timeout=glitchtip_instance.read_timeout,
+            max_retries=glitchtip_instance.max_retries,
+        ) as glitchtip_client:
+            current_state = fetch_current_state(
+                glitchtip_client=glitchtip_client,
+                # the automation user isn't managed by app-interface (chicken - egg problem), so just ignore it
+                ignore_users=[
+                    secret_reader.read_secret(glitchtip_instance.automation_user_email)
+                ],
+            )
+            desired_state = fetch_desired_state(
+                glitchtip_projects=[
+                    p
+                    for p in glitchtip_projects
+                    if p.organization.instance.name == glitchtip_instance.name
+                ],
+                mail_domain=glitchtip_instance.mail_domain or "redhat.com",
+                internal_groups_client=internal_groups_client,
+            )
 
-        reconciler = GlitchtipReconciler(glitchtip_client, dry_run)
-        reconciler.reconcile(current_state, desired_state)
+            reconciler = GlitchtipReconciler(glitchtip_client, dry_run)
+            reconciler.reconcile(current_state, desired_state)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]: