qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/jira_permissions_validator.py

@@ -0,0 +1,340 @@
+import logging
+import sys
+from collections.abc import Callable, Iterable
+from enum import IntFlag, auto
+from typing import Any, TypedDict
+
+from jira import JIRAError
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.jira_permissions_validator.jira_boards_for_permissions_validator import (
+    JiraBoardV1,
+)
+from reconcile.gql_definitions.jira_permissions_validator.jira_boards_for_permissions_validator import (
+    query as query_jira_boards,
+)
+from reconcile.status import ExitCodes
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.jira_settings import get_jira_settings
+from reconcile.typed_queries.jiralert_settings import get_jiralert_settings
+from reconcile.utils import gql, metrics
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
+from reconcile.utils.jira_client import JiraClient, JiraWatcherSettings
+from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.unleash import get_feature_toggle_state
+
+QONTRACT_INTEGRATION = "jira-permissions-validator"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)
+
+NameToIdMap = dict[str, str]
+
+
+class BaseMetric(BaseModel):
+    """Base class for metrics"""
+
+    jira_server: str
+    board: str
+
+
+class PermissionErrorCounter(BaseMetric, metrics.GaugeMetric):
+    """Boards with permission errors."""
+
+    @classmethod
+    def name(cls) -> str:
+        return "jira_permissions_validator_permission_error"
+
+
+class ValidationError(IntFlag):
+    CANT_CREATE_ISSUE = auto()
+    CANT_TRANSITION_ISSUES = auto()
+    INVALID_ISSUE_TYPE = auto()
+    INVALID_ISSUE_STATE = auto()
+    INVALID_SECURITY_LEVEL = auto()
+    INVALID_PRIORITY = auto()
+    PERMISSION_ERROR = auto()
+    PUBLIC_PROJECT_NO_SECURITY_LEVEL = auto()
+    INVALID_COMPONENT = auto()
+    PROJECT_ARCHIVED = auto()
+
+
+class RunnerParams(TypedDict):
+    boards: list[JiraBoardV1]
+    dry_run: bool
+
+
+class CacheSource(TypedDict):
+    boards: list
+
+
+def board_is_valid(
+    jira: JiraClient,
+    board: JiraBoardV1,
+    default_issue_type: str,
+    default_reopen_state: str,
+    jira_server_priorities: NameToIdMap,
+    public_projects: Iterable[str],
+) -> ValidationError:
+    error = ValidationError(0)
+    try:
+        if jira.is_archived:
+            logging.error(f"[{board.name}] project is archived")
+            return ValidationError.PROJECT_ARCHIVED
+
+        if not jira.can_create_issues():
+            logging.error(f"[{board.name}] can not create issues in project")
+            error |= ValidationError.CANT_CREATE_ISSUE
+
+        if not jira.can_transition_issues():
+            logging.error(
+                f"[{board.name}] AppSRE Jira Bot user does not have the permission to change the issue status."
+            )
+            error |= ValidationError.CANT_TRANSITION_ISSUES
+
+        components = jira.components()
+        for escalation_policy in board.escalation_policies or []:
+            jira_component = escalation_policy.channels.jira_component
+            if jira_component and jira_component not in components:
+                logging.error(
+                    f"[{board.name}] escalation policy '{escalation_policy.name}' references a non existing Jira component "
+                    f"'{jira_component}'. Valid components: {components}"
+                )
+                error |= ValidationError.INVALID_COMPONENT
+
+        issue_type = board.issue_type if board.issue_type else default_issue_type
+        project_issue_types = jira.project_issue_types()
+        project_issue_types_str = [i.name for i in project_issue_types]
+        if issue_type not in project_issue_types_str:
+            logging.error(
+                f"[{board.name}] {issue_type} is not a valid issue type in project. Valid issue types: {project_issue_types_str}"
+            )
+            error |= ValidationError.INVALID_ISSUE_TYPE
+
+        available_states = []
+        for project_issue_type in project_issue_types:
+            if issue_type == project_issue_type.name:
+                available_states = project_issue_type.statuses
+                break
+
+        if not available_states:
+            logging.error(
+                f"[{board.name}] {issue_type} doesn't have any status. Choose a different issue type."
+            )
+            error |= ValidationError.INVALID_ISSUE_TYPE
+
+        reopen_state = (
+            board.issue_reopen_state
+            if board.issue_reopen_state
+            else default_reopen_state
+        )
+        if reopen_state.lower() not in [t.lower() for t in available_states]:
+            logging.error(
+                f"[{board.name}] '{reopen_state}' is not a valid state in project. Valid states: {available_states}"
+            )
+            error |= ValidationError.INVALID_ISSUE_STATE
+
+        if board.issue_resolve_state and board.issue_resolve_state.lower() not in [
+            t.lower() for t in available_states
+        ]:
+            logging.error(
+                f"[{board.name}] '{board.issue_resolve_state}' is not a valid state in project. Valid states: {available_states}"
+            )
+            error |= ValidationError.INVALID_ISSUE_STATE
+
+        if board.issue_security_id:
+            security_levels = jira.security_levels()
+            if board.issue_security_id not in [level.id for level in security_levels]:
+                logging.error(
+                    f"[{board.name}] {board.issue_security_id} is not a valid security level in project. Valid security ids: "
+                    + ", ".join([
+                        f"{level.name} - {level.id}" for level in jira.security_levels()
+                    ])
+                )
+                error |= ValidationError.INVALID_SECURITY_LEVEL
+        elif board.name in public_projects:
+            logging.error(
+                f"[{board.name}] is a public project, but no security level is defined."
+            )
+            error |= ValidationError.PUBLIC_PROJECT_NO_SECURITY_LEVEL
+
+        project_priorities = jira.project_priority_scheme()
+        # get the priority names from the project priorities ids
+        project_priorities_names = [
+            p_name
+            for project_p_id in project_priorities
+            for p_name, p_id in jira_server_priorities.items()
+            if p_id == project_p_id
+        ]
+        for priority in board.severity_priority_mappings.mappings:
+            if priority.priority not in jira_server_priorities:
+                logging.error(
+                    f"[{board.name}] {priority.priority} is not a valid Jira priority. Valid priorities: {project_priorities_names}"
+                )
+                error |= ValidationError.INVALID_PRIORITY
+                continue
+            if jira_server_priorities[priority.priority] not in project_priorities:
+                logging.error(
+                    f"[{board.name}] {priority.priority} is not a valid priority in project. Valid priorities: {project_priorities_names}"
+                )
+                error |= ValidationError.INVALID_PRIORITY
+    except JIRAError as e:
+        if e.status_code == 401:
+            # sporadic 401 errors, retrying
+            logging.debug(f"[{board.name}] sporadic 401 error! Retry later.")
+        elif e.status_code == 403:
+            logging.error(
+                f"[{board.name}] AppSRE Jira Bot user does not have all necessary permissions. Try granting the user the administrator permissions. API URL: {e.url}"
+            )
+            error |= ValidationError.PERMISSION_ERROR
+        else:
+            raise
+
+    return error
+
+
+def validate_boards(
+    metrics_container: metrics.MetricsContainer,
+    secret_reader: SecretReaderBase,
+    jira_client_settings: JiraWatcherSettings | None,
+    jira_boards: Iterable[JiraBoardV1],
+    default_issue_type: str,
+    default_reopen_state: str,
+    dry_run: bool,
+    jira_client_class: type[JiraClient] = JiraClient,
+) -> bool:
+    error = False
+    jira_clients: dict[str, JiraClient] = {}
+    for board in jira_boards:
+        logging.debug(f"[{board.name}] checking ...")
+        if board.server.server_url not in jira_clients:
+            jira_clients[board.server.server_url] = jira_client_class.create(
+                project_name=board.name,
+                token=secret_reader.read_secret(board.server.token),
+                server_url=board.server.server_url,
+                jira_watcher_settings=jira_client_settings,
+            )
+        jira = jira_clients[board.server.server_url]
+        jira.project = board.name
+        try:
+            error_flags = board_is_valid(
+                jira=jira,
+                board=board,
+                default_issue_type=default_issue_type,
+                default_reopen_state=default_reopen_state,
+                jira_server_priorities={p.name: p.id for p in jira.priorities()},
+                public_projects=jira.public_projects(),
+            )
+            match error_flags:
+                case 0:
+                    # no errors
+                    logging.debug(f"[{board.name}] is valid")
+                case ValidationError.PERMISSION_ERROR:
+                    # we don't have all the permissions, but we can create jira tickets
+                    metrics_container.set_gauge(
+                        PermissionErrorCounter(
+                            jira_server=board.server.server_url,
+                            board=board.name,
+                        ),
+                        value=1,
+                    )
+                    if dry_run:
+                        # throw an error for MR checks but not in prod mode
+                        error = True
+                case (
+                    ValidationError.CANT_CREATE_ISSUE | ValidationError.PROJECT_ARCHIVED
+                ):
+                    if dry_run:
+                        # throw an error for MR checks but not in prod mode
+                        error = True
+                case _:
+                    error = True
+        except Exception as e:
+            logging.error(f"[{board.name}] {e}")
+            error = True
+    return error
+
+
+def get_jira_boards(query_func: Callable) -> list[JiraBoardV1]:
+    return [
+        board
+        for board in query_jira_boards(query_func=query_func).jira_boards or []
+        if integration_is_enabled(QONTRACT_INTEGRATION, board)
+    ]
+
+
+def export_boards(boards: list[JiraBoardV1]) -> list[dict]:
+    return [board.dict() for board in boards]
+
+
+def run(
+    dry_run: bool,
+    enable_extended_early_exit: bool = False,
+    extended_early_exit_cache_ttl_seconds: int = 3600,
+    log_cached_log_output: bool = False,
+) -> None:
+    gql_api = gql.get_api()
+    boards = get_jira_boards(query_func=gql_api.query)
+    runner_params: RunnerParams = {
+        "boards": boards,
+        "dry_run": dry_run,
+    }
+    if enable_extended_early_exit and get_feature_toggle_state(
+        "jira-permissions-validator-extended-early-exit",
+        default=True,
+    ):
+        vault_settings = get_app_interface_vault_settings()
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
+        cache_source = CacheSource(
+            boards=export_boards(boards),
+        )
+        extended_early_exit_run(
+            integration=QONTRACT_INTEGRATION,
+            integration_version=QONTRACT_INTEGRATION_VERSION,
+            # don't use `dry_run` in the cache key because this is a read-only integration
+            dry_run=False,
+            cache_source=cache_source,
+            shard="",
+            ttl_seconds=extended_early_exit_cache_ttl_seconds,
+            logger=logging.getLogger(),
+            runner=runner,
+            runner_params=runner_params,
+            secret_reader=secret_reader,
+            log_cached_log_output=log_cached_log_output,
+        )
+    else:
+        runner(**runner_params)
+
+
+def runner(boards: list[JiraBoardV1], dry_run: bool) -> ExtendedEarlyExitRunnerResult:
+    gql_api = gql.get_api()
+    settings = get_jira_settings(gql_api=gql_api)
+    jiralert_settings = get_jiralert_settings(query_func=gql_api.query)
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
+    with metrics.transactional_metrics("jira-boards") as metrics_container:
+        error = validate_boards(
+            metrics_container=metrics_container,
+            secret_reader=secret_reader,
+            jira_client_settings=settings.jira_watcher,
+            jira_boards=boards,
+            default_issue_type=jiralert_settings.default_issue_type,
+            default_reopen_state=jiralert_settings.default_reopen_state,
+            dry_run=dry_run,
+        )
+
+    if error:
+        sys.exit(ExitCodes.ERROR)
+
+    return ExtendedEarlyExitRunnerResult(payload=export_boards(boards), applied_count=0)
+
+
+def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
+    return {"boards": export_boards(get_jira_boards(query_func=gql.get_api().query))}

reconcile/jira_watcher.py

@@ -34,8 +34,8 @@ def format_message(server, key, data, event, previous_state=None, current_state=
         if previous_state and current_state
         else ""
     )
-    url = "{}/browse/{}".format(server, key)
-    return "{} ({}) {}{}".format(url, summary, event, info)
+    url = f"{server}/browse/{key}"
+    return f"{url} ({summary}) {event}{info}"
 
 
 def calculate_diff(server, current_state, previous_state):
@@ -55,9 +55,7 @@ def calculate_diff(server, current_state, previous_state):
     messages.extend(deleted_issues)
 
     updated_issues = [
-        format_message(
-            server, key, data, "status change", previous_state[key], current_state[key]
-        )
+        format_message(server, key, data, "status change", previous_state[key], data)
         for key, data in current_state.items()
         if key in previous_state and data["status"] != previous_state[key]["status"]
     ]

reconcile/ldap_groups/integration.py

@@ -0,0 +1,279 @@
+import contextlib
+import logging
+from collections.abc import (
+    Callable,
+    Iterable,
+)
+from typing import Any
+
+from reconcile.gql_definitions.ldap_groups.roles import RoleV1
+from reconcile.gql_definitions.ldap_groups.roles import query as roles_query
+from reconcile.gql_definitions.ldap_groups.settings import LdapGroupsSettingsV1
+from reconcile.gql_definitions.ldap_groups.settings import query as settings_query
+from reconcile.utils import gql
+from reconcile.utils.aws_helper import unique_sso_aws_accounts
+from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_iterables
+from reconcile.utils.exceptions import (
+    AppInterfaceLdapGroupsSettingsError,
+    AppInterfaceSettingsError,
+)
+from reconcile.utils.helpers import find_duplicates
+from reconcile.utils.internal_groups.client import (
+    InternalGroupsClient,
+    NotFound,
+)
+from reconcile.utils.internal_groups.models import (
+    Entity,
+    EntityType,
+    Group,
+)
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.state import (
+    State,
+    init_state,
+)
+
+QONTRACT_INTEGRATION = "ldap-groups"
+
+
+class LdapGroupsIntegrationParams(PydanticRunParams):
+    aws_sso_namespace: str
+
+
+class LdapGroupsIntegration(QontractReconcileIntegration[LdapGroupsIntegrationParams]):
+    """Manage LDAP groups based on App-Interface roles."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(
+        self, query_func: Callable | None = None
+    ) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        if not query_func:
+            query_func = gql.get_api().query
+        return {"roles": [c.dict() for c in self.get_roles(query_func)]}
+
+    @defer
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
+        """Run the integration."""
+        gql_api = gql.get_api()
+        roles = self.get_roles(gql_api.query)
+        if not roles:
+            logging.debug("No roles found.")
+            return
+        self.settings = self.get_integration_settings(gql_api.query)
+        secret = self.secret_reader.read_all_secret(self.settings.credentials)
+
+        # APIs
+        state_obj = init_state(integration=self.name, secret_reader=self.secret_reader)
+        internal_groups_client = InternalGroupsClient(
+            secret["api_url"],
+            secret["issuer_url"],
+            secret["client_id"],
+            secret["client_secret"],
+        )
+        if defer:
+            defer(state_obj.cleanup)
+            defer(internal_groups_client.close)
+
+        # run
+        owner = Entity(
+            type=EntityType.SERVICE_ACCOUNT,
+            # OIDC service accounts are named service-account-<client_id>
+            id=f'service-account-{secret["client_id"]}',
+        )
+        desired_groups_for_roles = self.get_desired_groups_for_roles(
+            roles,
+            contact_list=self.settings.contact_list,
+            default_owners=[owner],
+        )
+        desired_groups_for_aws_roles = self.get_desired_groups_for_aws_roles(
+            roles,
+            contact_list=self.settings.contact_list,
+            default_owners=[owner],
+        )
+        desired_groups = desired_groups_for_roles + desired_groups_for_aws_roles
+
+        group_names = self.get_managed_groups(state_obj)
+        # take desired groups into account to support overtaking existing ones
+        group_names.update(g.name for g in desired_groups)
+        current_groups = self.fetch_current_state(
+            internal_groups_client, group_names=group_names
+        )
+        try:
+            self.reconcile(
+                dry_run=dry_run,
+                internal_groups_client=internal_groups_client,
+                desired_groups=desired_groups,
+                current_groups=current_groups,
+            )
+        finally:
+            self.set_managed_groups(dry_run=dry_run, state_obj=state_obj)
+
+    def get_managed_groups(self, state_obj: State) -> set[str]:
+        """Return the managed groups from the state object."""
+        try:
+            return set(state_obj["managed_groups"])
+        except KeyError:
+            return set()
+
+    def set_managed_groups(self, dry_run: bool, state_obj: State) -> None:
+        """Set the managed groups in the state object."""
+        if not dry_run:
+            state_obj["managed_groups"] = sorted(self._managed_groups)
+
+    @staticmethod
+    def get_integration_settings(query_func: Callable) -> LdapGroupsSettingsV1:
+        """Return the integration settings."""
+        data = settings_query(query_func)
+        if not data.settings:
+            raise AppInterfaceSettingsError("No app-interface settings found.")
+        if not data.settings[0].ldap_groups:
+            raise AppInterfaceLdapGroupsSettingsError(
+                "No app-interface ldap-groups settings found."
+            )
+        return data.settings[0].ldap_groups
+
+    def get_roles(self, query_func: Callable) -> list[RoleV1]:
+        """Return the roles with ldap_group set."""
+        data = roles_query(query_func, variables={})
+        roles = list(data.roles or [])
+        if duplicates := find_duplicates(
+            role.ldap_group.name for role in roles if role.ldap_group
+        ):
+            for dup in duplicates:
+                logging.error(f"{dup} is already in use by another role.")
+            raise ValueError("Duplicate ldapGroup value(s) found.")
+        return roles
+
+    def get_desired_groups_for_roles(
+        self,
+        roles: Iterable[RoleV1],
+        default_owners: list[Entity],
+        contact_list: str,
+    ) -> list[Group]:
+        """Return the desired rover groups for the given roles."""
+        groups = []
+        for role in roles:
+            if not role.ldap_group:
+                continue
+            members = [
+                Entity(type=EntityType.USER, id=user.org_username)
+                for user in role.users
+            ]
+            groups.append(
+                Group(
+                    name=role.ldap_group.name,
+                    description="Persisted App-Interface role. Managed by qontract-reconcile",
+                    notes=role.ldap_group.notes,
+                    display_name=f"{role.ldap_group.name} (App-Interface))",
+                    members=members,
+                    owners=default_owners
+                    if not role.ldap_group.members_are_owners
+                    else default_owners + members,
+                    contact_list=contact_list,
+                )
+            )
+        return groups
+
+    def get_desired_groups_for_aws_roles(
+        self,
+        roles: Iterable[RoleV1],
+        default_owners: Iterable[Entity],
+        contact_list: str,
+    ) -> list[Group]:
+        """Return the desired rover groups for all AWS roles."""
+        groups = []
+        for role in roles:
+            if not role.users or (not role.aws_groups and not role.user_policies):
+                continue
+            user_policies = role.user_policies or []
+            aws_groups = role.aws_groups or []
+            for account in unique_sso_aws_accounts(
+                integration=self.name,
+                accounts=[i.account for i in user_policies + aws_groups],
+            ):
+                group_name = (
+                    f"{self.params.aws_sso_namespace}-{account.uid}-{role.name}"
+                )
+                groups.append(
+                    Group(
+                        name=group_name,
+                        description=f"AWS account: '{account.name}' Role: '{role.name}' Managed by qontract-reconcile",
+                        display_name=group_name,
+                        members=[
+                            Entity(type=EntityType.USER, id=user.org_username)
+                            for user in role.users
+                        ],
+                        # only owners can modify the group (e.g. add/remove members)
+                        owners=default_owners,
+                        contact_list=contact_list,
+                    )
+                )
+
+        return groups
+
+    def fetch_current_state(
+        self, internal_groups_client: InternalGroupsClient, group_names: Iterable[str]
+    ) -> list[Group]:
+        """Reach out to the internal groups API and fetch all managed groups."""
+        groups = []
+        for group_name in group_names:
+            with contextlib.suppress(NotFound):
+                groups.append(internal_groups_client.group(group_name))
+        return groups
+
+    def reconcile(
+        self,
+        dry_run: bool,
+        internal_groups_client: InternalGroupsClient,
+        desired_groups: Iterable[Group],
+        current_groups: Iterable[Group],
+    ) -> None:
+        """Reach out to the internal groups API and reconcile the groups."""
+        diff_result = diff_iterables(
+            current_groups,
+            desired_groups,
+            key=lambda g: g.name,
+            equal=lambda g1, g2: g1 == g2,
+        )
+        # Internal Groups API does not support listing all managed groups, therefore
+        # we need to keep track of them ourselves.
+        self._managed_groups = {g.name for g in current_groups}
+
+        for group_to_add in diff_result.add.values():
+            logging.info([
+                "create_ldap_group",
+                group_to_add.name,
+                f"members={', '.join(u.id for u in group_to_add.members)}",
+                f"owners={', '.join(u.id for u in group_to_add.owners)}",
+                f"notes={group_to_add.notes}",
+            ])
+            if not dry_run:
+                internal_groups_client.create_group(group_to_add)
+                self._managed_groups.add(group_to_add.name)
+
+        for group_to_remove in diff_result.delete.values():
+            logging.info(["delete_ldap_group", group_to_remove.name])
+            if not dry_run:
+                with contextlib.suppress(NotFound):
+                    internal_groups_client.delete_group(group_to_remove.name)
+                self._managed_groups.remove(group_to_remove.name)
+
+        for diff_pair in diff_result.change.values():
+            group_to_update = diff_pair.desired
+            logging.info([
+                "update_ldap_group",
+                group_to_update.name,
+                f"members={', '.join(u.id for u in group_to_update.members)}",
+                f"owners={', '.join(u.id for u in group_to_update.owners)}",
+                f"notes={group_to_update.notes}",
+            ])
+            if not dry_run:
+                internal_groups_client.update_group(group_to_update)

reconcile/ldap_users.py

@@ -37,6 +37,9 @@ def init_users() -> list[dict[str, list]]:
         for a in user.get("aws_accounts", []):
             item = {"type": PathTypes.AWS_ACCOUNTS, "path": "data" + a["path"]}
             users[u].append(item)
+        for s in user.get("schedules"):
+            item = {"type": PathTypes.SCHEDULE, "path": "data" + s["path"]}
+            users[u].append(item)
 
     return [{"username": username, "paths": paths} for username, paths in users.items()]