gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/processors/project/members_processor.py

@@ -0,0 +1,204 @@
+import sys
+from logging import critical, info, warning, error
+from gitlab import GitlabDeleteError
+from gitlab.v4.objects import Project, User
+
+from gitlabform.constants import EXIT_INVALID_INPUT
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class MembersProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("members", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        keep_bots = configuration.get("members|keep_bots", False)
+
+        enforce_members = configuration.get("members|enforce", False)
+
+        groups = configuration.get("members|groups", {})
+
+        users = configuration.get("members|users", {})
+
+        if not groups and not users and not enforce_members:
+            critical(
+                "Project members configuration section has to contain"
+                " either 'users' or 'groups' non-empty keys"
+                " (unless you want to enforce no direct members)."
+            )
+            sys.exit(EXIT_INVALID_INPUT)
+
+        self._process_groups(project_and_group, groups, enforce_members)
+        self._process_users(project_and_group, users, enforce_members, keep_bots)
+
+    def _process_groups(self, project_and_group: str, groups: dict, enforce_members: bool):
+        if groups:
+            info("Processing groups as members...")
+
+            current_groups = self.gitlab.get_groups_from_project(project_and_group)
+            for group in groups:
+                info(f"Processing group {group}...")
+                expires_at = groups[group]["expires_at"].strftime("%Y-%m-%d") if "expires_at" in groups[group] else None
+                access_level = groups[group]["group_access"] if "group_access" in groups[group] else None
+
+                # we only add the group if it doesn't have the correct settings
+                common_group_name = group.lower()
+                if (
+                    common_group_name in current_groups
+                    and expires_at == current_groups[common_group_name]["expires_at"]
+                    and access_level == current_groups[common_group_name]["group_access_level"]
+                ):
+                    info(f"Ignoring group '{common_group_name}' as it is already a member")
+                    info(f"Current settings for '{common_group_name}' are: {current_groups[common_group_name]}")
+                else:
+                    info(f"Setting group '{common_group_name}' as a member")
+                    access = access_level
+                    expiry = expires_at
+
+                    # we will remove group access first and then re-add them,
+                    # to ensure that the groups have the expected access level
+                    self.gitlab.unshare_with_group(project_and_group, common_group_name)
+                    self.gitlab.share_with_group(project_and_group, common_group_name, access, expiry)
+
+        if enforce_members:
+            current_groups = self.gitlab.get_groups_from_project(project_and_group)
+
+            groups_in_config = [group_name.lower() for group_name in groups.keys()]
+            groups_in_gitlab = current_groups.keys()
+            groups_not_in_config = set(groups_in_gitlab) - set(groups_in_config)
+
+            for group_not_in_config in groups_not_in_config:
+                info(f"Removing group '{group_not_in_config}' that is not configured to be a member.")
+                self.gitlab.unshare_with_group(project_and_group, group_not_in_config)
+        else:
+            info("Not enforcing group members.")
+
+    def _process_users(
+        self,
+        project_and_group: str,
+        users: dict,
+        enforce_members: bool,
+        keep_bots: bool,
+    ):
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+
+        current_members = self._get_members_from_project(project)
+
+        if users:
+            info("Processing users as members...")
+
+            for user in users:
+                info(f"Processing user '{user}'...")
+
+                gitlab_user = self.gl.get_user_by_username_cached(user)
+                if gitlab_user is None:
+                    warning(f"Could not find user '{user}' in Gitlab, skipping...")
+                    continue
+                user_id = gitlab_user.id
+
+                expires_at = users[user]["expires_at"].strftime("%Y-%m-%d") if "expires_at" in users[user] else None
+                access_level = users[user]["access_level"] if "access_level" in users[user] else None
+                member_role_id_or_name = users[user]["member_role"] if "member_role" in users[user] else None
+                if member_role_id_or_name:
+                    group_name_and_path = project.namespace["full_path"]
+                    member_role_id = self.gl.get_member_role_id_cached(member_role_id_or_name, group_name_and_path)
+                else:
+                    member_role_id = None
+
+                # we only add the user if it doesn't have the correct settings.
+                # To make sure that the user hasn't been added in a different
+                # case, we enforce that the username is always in lowercase for
+                # checks.
+                common_username = user.lower()
+
+                if common_username in current_members:
+                    current_member = current_members[common_username]
+                    if hasattr(current_member, "member_role"):
+                        member_role_id_before = current_member.member_role["id"]
+                    else:
+                        member_role_id_before = None
+                    if (
+                        expires_at == current_member.expires_at
+                        and access_level == current_member.access_level
+                        and member_role_id == member_role_id_before
+                    ):
+                        info(f"Nothing to change for user '{common_username}' - same config now as to set.")
+                        info(f"Current settings for '{common_username}' are: {current_members[common_username]}")
+                    else:
+                        info(
+                            f"Editing user '{common_username}' membership to change their access level or expires at",
+                        )
+
+                        project_member = project.members.get(id=user_id)
+                        project_member.access_level = access_level
+                        project_member.member_role_id = member_role_id_before
+                        if expires_at:
+                            project_member.expires_at = expires_at
+                        project_member.save()
+
+                else:
+                    info(
+                        f"Adding user '{common_username}' who previously was not a member.",
+                    )
+                    create_data = {
+                        "user_id": user_id,
+                        "access_level": access_level,
+                    }
+
+                    if expires_at:
+                        create_data["expires_at"] = expires_at
+
+                    if member_role_id:
+                        create_data["member_role_id"] = member_role_id
+
+                    project.members.create(data=create_data)
+
+        if enforce_members:
+            info("Enforcing Project members")
+            # Enforce that all usernames are lowercase for comparisons.
+            users_in_config = [username.lower() for username in users.keys()]
+            users_in_gitlab = current_members.keys()
+            users_not_in_config = set(users_in_gitlab) - set(users_in_config)
+            for user_not_in_config in users_not_in_config:
+                info(f"Removing user '{user_not_in_config}' that is not configured to be a member.")
+                gl_user: User | None = self.gl.get_user_by_username_cached(user_not_in_config)
+
+                if gl_user is None:
+                    # User does not exist an instance level but is for whatever reason present on a Group/Project
+                    # We should raise error into Logs but not prevent the rest of GitLabForm from executing
+                    # This error is more likely to be prevalent in Dedicated instances; it is unlikely for a User to
+                    # be completely deleted from gitlab.com
+                    error(
+                        f"Could not find User '{user_not_in_config}' on the Instance so can not remove User from "
+                        f"Project '{project_and_group}'"
+                    )
+                    continue
+
+                if keep_bots and gl_user.bot:
+                    info(f"Will not remove bot user '{user_not_in_config}' as the 'keep_bots' option is true.")
+                    continue
+
+                try:
+                    project.members.delete(id=gl_user.id)
+                except GitlabDeleteError as delete_error:
+                    error(f"Member '{user_not_in_config}' could not be deleted: {delete_error}")
+                    raise delete_error
+        else:
+            info("Not enforcing user members.")
+
+    @staticmethod
+    def _get_members_from_project(project):
+        # Only get direct members from Python Gitlab (matches previous implementation)
+        # https://python-gitlab.readthedocs.io/en/stable/gl_objects/projects.html#id14
+        project_members = project.members.list(iterator=True)
+        existing_user_names_lower = dict()
+
+        for member in project_members:
+            # To make sure that the user hasn't been added in a different
+            # case, we enforce that the username is always in lowercase for
+            # checks.
+            existing_user_names_lower[member.username.lower()] = member
+
+        info(f"Existing project members {existing_user_names_lower}")
+        return existing_user_names_lower

gitlabform/processors/project/merge_requests_approval_rules.py

@@ -0,0 +1,17 @@
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.defining_keys import Key, And
+from gitlabform.processors.multiple_entities_processor import MultipleEntitiesProcessor
+
+
+class MergeRequestsApprovalRules(MultipleEntitiesProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__(
+            "merge_requests_approval_rules",
+            gitlab,
+            list_method_name="get_approval_rules",
+            add_method_name="add_approval_rule",
+            edit_method_name="edit_approval_rule",
+            delete_method_name="delete_approval_rule",
+            defining=Key("name"),
+            required_to_create_or_update=And(Key("name"), Key("approvals_required")),
+        )

gitlabform/processors/project/merge_requests_approvals.py

@@ -0,0 +1,59 @@
+import sys
+from logging import debug, info, critical
+from typing import Callable
+from gitlab.v4.objects import Project
+
+from gitlabform import EXIT_INVALID_INPUT
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.util.difference_logger import DifferenceLogger
+
+
+class MergeRequestsApprovals(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("merge_requests_approvals", gitlab)
+        self.get_entity_in_gitlab: Callable = getattr(self, "get_project_mr_approvals_settings")
+
+    def _process_configuration(self, project_path: str, configuration: dict) -> None:
+        info("Processing project merge requests approvals settings...")
+        project: Project = self.gl.get_project_by_path_cached(project_path)
+
+        mr_approval_settings_in_config: dict = configuration.get("merge_requests_approvals", {})
+        mr_approval_settings_in_gitlab: dict = project.approvals.get().asdict()
+
+        info(mr_approval_settings_in_gitlab)
+        info("merge_requests_approvals BEFORE: ^^^")
+
+        if self._needs_update(mr_approval_settings_in_gitlab, mr_approval_settings_in_config):
+            debug("Updating project merge requests approvals settings")
+            project.approvals.update(**mr_approval_settings_in_config)
+
+            info(project.approvals.get().asdict())
+            info("merge_requests_approvals AFTER: ^^^")
+        else:
+            info("No update needed for project merge requests approvals settings")
+
+    def get_project_mr_approvals_settings(self, project_path: str):
+        return self.gl.get_project_by_path_cached(project_path).approvals.get().asdict()
+
+    def _can_proceed(self, project_or_group: str, configuration: dict):
+        if "approvals_before_merge" in configuration["merge_requests_approvals"]:
+            critical(
+                "Setting the 'approvals_before_merge' in the 'merge_requests_approvals' sections is not allowed "
+                "as it is not clear which rule does it apply to. "
+                "Please set it inside the specific approval rules in the 'merge_requests_approval_rules' section."
+            )
+            sys.exit(EXIT_INVALID_INPUT)
+        else:
+            return True
+
+    # TODO: duplicated logic with project_settings_processor.py. Should be refactored - ideally in the AbstractProcessor
+    def _print_diff(self, project_or_project_and_group: str, entity_config, diff_only_changed: bool):
+        entity_in_gitlab = self.get_project_mr_approvals_settings(project_or_project_and_group)
+
+        DifferenceLogger.log_diff(
+            f"{self.configuration_name} changes",
+            entity_in_gitlab,
+            entity_config,
+            only_changed=diff_only_changed,
+        )

gitlabform/processors/project/project_labels_processor.py

@@ -0,0 +1,27 @@
+from typing import Dict
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+from gitlab.v4.objects import Project
+
+from gitlabform.processors.util.labels_processor import LabelsProcessor
+
+
+class ProjectLabelsProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("labels", gitlab)
+        self._labels_processor = LabelsProcessor()
+
+    def _process_configuration(self, project_and_group: str, configuration: Dict):
+        configured_labels = configuration.get("labels", {})
+
+        enforce = configuration.get("labels|enforce", False)
+
+        # Remove 'enforce' key from the config so that it's not treated as a "label"
+        if enforce:
+            configured_labels.pop("enforce")
+
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+
+        self._labels_processor.process_labels(configured_labels, enforce, project, self._needs_update)

gitlabform/processors/project/project_processor.py

@@ -0,0 +1,62 @@
+from logging import info, error
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlab import GitlabGetError, GitlabTransferProjectError
+from gitlab.v4.objects import Project
+
+
+class ProjectProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("project", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        project_path_with_namespace: str = project_and_group
+
+        if configuration["project"].get("transfer_from") is not None:
+            source_project_path_with_namespace = configuration["project"].get("transfer_from")
+
+            # Check if the project was already transfered (i.e. in previous run) or a project with same path already exists
+            try:
+                project_in_config: Project = self.gl.get_project_by_path_cached(project_path_with_namespace)
+                info(
+                    f"Project already exists: '{project_in_config.path_with_namespace}'. Ignoring 'transfer_from' config..."
+                )
+            except GitlabGetError:
+                # Project doesn't exist at the destination. Let's process the transfer request
+                project_to_be_transferred: Project = self.gl.get_project_by_path_cached(
+                    source_project_path_with_namespace
+                )
+                destination_project_path = project_and_group.split("/")[-1]
+                # Check if the project path needs to be updated; In Gitlab, path maybe different than name
+                if destination_project_path != project_to_be_transferred.path:
+                    info(
+                        f"Updating the source project path from '{project_to_be_transferred.path}' to '{destination_project_path}'"
+                    )
+                    self.gl.projects.update(project_to_be_transferred.id, {"path": destination_project_path})
+
+                # TODO: Catch GitlabTransferProjectError exception.
+                #  See the next comment for details.
+                # try:
+                project_transfer_destination_group, _ = project_and_group.rsplit("/", 1)
+                info(f"Transferring project to '{project_transfer_destination_group}' group...")
+                project_to_be_transferred.transfer(project_transfer_destination_group)
+                # TODO: Catch GitlabTransferProjectError exception.
+                #  The above code can run into exception for various reasons.
+                #  We should catch this exception and log a custom error message with hints.
+                #  In some scenarios we want to break and in some we want to proceed and attempt a transfer regardless
+                #  For more details, see: https://github.com/gitlabform/gitlabform/issues/611
+                # except GitlabTransferProjectError as e:
+                #     critical(
+                #         "Encountered error transferring project. Please check if project transfer requirements were met. Docs: https://docs.gitlab.com/ee/user/project/settings/index.html#transfer-a-project-to-another-namespace"
+                #     )
+                #     raise
+
+        if configuration["project"].get("archive") is not None:
+            project: Project = self.gl.get_project_by_path_cached(project_path_with_namespace)
+
+            if configuration["project"].get("archive") is True:
+                info("Archiving project...")
+                project.archive()
+            elif configuration["project"].get("archive") is False:
+                info("Unarchiving project...")
+                project.unarchive()

gitlabform/processors/project/project_push_rules_processor.py

@@ -0,0 +1,52 @@
+from logging import debug
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlab.v4.objects.projects import Project
+from gitlab.exceptions import GitlabGetError, GitlabParsingError
+
+
+class ProjectPushRulesProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("project_push_rules", gitlab)
+
+    def _process_configuration(self, project_path: str, configuration: dict):
+        configured_project_push_rules = configuration.get("project_push_rules", {})
+        project: Project = self.gl.get_project_by_path_cached(project_path)
+
+        try:
+            existing_push_rules = project.pushrules.get()
+        except GitlabGetError as e:
+            if e.response_code == 404:
+                debug(f"No existing push rules for '{project.name}', creating new push rules.")
+                self.create_project_push_rules(project, configured_project_push_rules)
+                return
+        except GitlabParsingError as e:
+            # Known issue from GitLab API when project push rule has never been configured.
+            # In that scenario, the API returns `null` instead of an appropriate message
+            # and error code.
+            # See GitLab issue : https://gitlab.com/gitlab-org/gitlab/-/issues/513331
+            # Until the issue is resolved, we will assume push rule is not configured.
+            # When the above issue is resolved, this exception block can be removed.
+
+            debug(
+                f"Cannot determine if push rule is currently configured for '{project.name}', attempting to create new push rules."
+            )
+            self.create_project_push_rules(project, configured_project_push_rules)
+            return
+
+        if self._needs_update(existing_push_rules.asdict(), configured_project_push_rules):
+            self.update_push_rules(existing_push_rules, configured_project_push_rules)
+        else:
+            debug("No update needed for Project Push Rules")
+
+    @staticmethod
+    def update_push_rules(push_rules, configured_project_push_rules: dict):
+        for key, value in configured_project_push_rules.items():
+            debug(f"Updating setting {key} to value {value}")
+            setattr(push_rules, key, value)
+        push_rules.save()
+
+    @staticmethod
+    def create_project_push_rules(project, push_rules_config: dict):
+        debug(f"Creating push rules with configuration: {push_rules_config}")
+        project.pushrules.create(push_rules_config)

gitlabform/processors/project/project_security_settings.py

@@ -0,0 +1,66 @@
+from logging import debug, warning
+from typing import Any, cast
+from gitlab.v4.objects import Project
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.util.difference_logger import DifferenceLogger
+
+
+class ProjectSecuritySettingsProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("project_security_settings", gitlab)
+
+    def _process_configuration(self, project_name: str, configuration: dict) -> None:
+        project: Project = self.gl.get_project_by_path_cached(project_name)
+
+        security_settings_in_config = configuration.get("project_security_settings", {})
+        security_settings_in_gitlab = self.get_project_security_settings(project)
+
+        if self._needs_update(security_settings_in_gitlab, security_settings_in_config):
+            debug("Updating project security settings")
+            self._update_project_security_settings(project, security_settings_in_gitlab, security_settings_in_config)
+        else:
+            debug("No update needed for project security settings")
+
+    def get_project_security_settings(self, project: Project) -> dict:
+        """Retrieve project security settings using python-gitlab."""
+        try:
+            # TODO: python-gitlab does not yet support retrieving project security settings
+            # via its dedicated manager, so we use a lower-level http_get method here.
+            # Switch to native method once supported by python-gitlab.
+
+            path = f"/projects/{project.id}/security_settings"
+            result = self.gl.http_get(path)
+            # http_get can return Response for streamed requests, but we're not streaming
+            # so it will always be a dict
+            return cast(dict[str, Any], result)
+        except Exception as e:
+            warning(f"Failed to get project security settings for project {project.path_with_namespace}: {e}")
+            return {}
+
+    def _update_project_security_settings(self, project: Project, initial_settings: dict, settings: dict) -> None:
+        """Update project security settings using python-gitlab."""
+        try:
+            # TODO: python-gitlab does not yet support updating project security settings
+            # via its dedicated manager, so we use a lower-level http_put method here.
+            # Switch to native method once supported by python-gitlab.
+            debug(initial_settings)
+            debug("project_security_settings BEFORE: ^^^")
+            path = f"/projects/{project.id}/security_settings"
+            updated_security_config = self.gl.http_put(path, post_data=settings)
+            debug(cast(dict[str, Any], updated_security_config))
+            debug("project_security_settings AFTER: ^^^")
+        except Exception as e:
+            warning(f"Failed to update project security settings for project {project.path_with_namespace}: {e}")
+
+    def _print_diff(self, project_or_project_and_group: str, entity_config, diff_only_changed: bool):
+        project = self.gl.get_project_by_path_cached(project_or_project_and_group)
+        entity_in_gitlab = self.get_project_security_settings(project)
+
+        DifferenceLogger.log_diff(
+            f"{self.configuration_name} changes",
+            entity_in_gitlab,
+            entity_config,
+            only_changed=diff_only_changed,
+        )