gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/processors/project/files_processor.py

@@ -0,0 +1,301 @@
+import os
+import re
+import sys
+from pathlib import Path
+
+from logging import debug, info, warning, critical
+from typing import List
+
+from jinja2 import Environment, FileSystemLoader
+from gitlab import GitlabGetError, GitlabUpdateError
+from gitlab.v4.objects import Project, ProjectFile, ProjectBranch
+from gitlab.base import RESTObject
+
+from gitlabform.constants import EXIT_INVALID_INPUT, EXIT_PROCESSING_ERROR
+from gitlabform.configuration import Configuration
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.project.branches_processor import BranchesProcessor
+
+
+class FilesProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab, config: Configuration, strict: bool):
+        super().__init__("files", gitlab)
+        self.config = config
+        self.strict = strict
+        self.branch_processor = BranchesProcessor(gitlab, strict)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        for file in sorted(configuration["files"]):
+            project: Project = self.gl.get_project_by_path_cached(project_and_group)
+            debug("Processing file '%s'...", file)
+
+            if configuration.get("files|" + file + "|skip"):
+                debug("Skipping file '%s'", file)
+                continue
+
+            config_target_ref = configuration["files"][file]["branches"]
+            branches_to_update: List[RESTObject] = []
+
+            if isinstance(config_target_ref, str):
+                # Target ref could be either 'all' or 'protected'.
+                # Get a list of branches that should be updated.
+                if config_target_ref == "all":
+                    branches_to_update.extend(project.branches.list(get_all=True, lazy=True))
+                elif config_target_ref == "protected":
+                    branches_to_update.extend(project.protectedbranches.list(get_all=True, lazy=True))
+            elif isinstance(config_target_ref, list):
+                # Get a list of branches from the config that should be updated.
+                for branch_name in config_target_ref:
+                    try:
+                        branches_to_update.append(project.branches.get(branch_name))
+                    except GitlabGetError:
+                        message = f"! Branch '{branch_name}' not found, not processing file '{file}' in it"
+                        if self.strict:
+                            critical(message)
+                            sys.exit(EXIT_INVALID_INPUT)
+                        else:
+                            warning(message)
+
+            debug(
+                "File '%s' to be updated in '%s' branche(s)",
+                file,
+                len(branches_to_update),
+            )
+
+            for branch in branches_to_update:
+                info(f"Processing file '{file}' in branch '{branch.name}'")
+
+                if configuration.get("files|" + file + "|content") and configuration.get("files|" + file + "|file"):
+                    critical(
+                        f"File '{file}' in '{project_and_group}' has both `content` and `file` set - "
+                        "use only one of these keys."
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+                if configuration.get("files|" + file + "|delete"):
+                    try:
+                        file_to_delete: ProjectFile = project.files.get(file_path=file, ref=branch.name)
+                        debug("Deleting file '%s' in branch '%s'", file, branch.name)
+                        self.modify_file_dealing_with_branch_protection(
+                            project,
+                            branch,
+                            file_to_delete,
+                            "delete",
+                            configuration,
+                        )
+                    except GitlabGetError:
+                        debug(
+                            "Not deleting file '%s' in branch '%s' (already doesn't exist)",
+                            file,
+                            branch.name,
+                        )
+                else:
+                    # change or create file
+
+                    if configuration.get("files|" + file + "|content"):
+                        new_content = configuration.get("files|" + file + "|content")
+                    else:
+                        path_in_config = Path(str(configuration.get("files|" + file + "|file")))
+                        if path_in_config.is_absolute():
+                            effective_path = path_in_config
+                        else:
+                            # relative paths are relative to config file location
+                            effective_path = Path(os.path.join(self.config.config_dir, str(path_in_config)))
+                        new_content = effective_path.read_text()
+
+                    # templating is documented to be enabled by default,
+                    # see https://gitlabform.github.io/gitlabform/reference/files/#files
+                    templating_enabled = True
+
+                    if configuration.get("files|" + file + "|template", templating_enabled):
+                        new_content = self.get_file_content_as_template(
+                            new_content,
+                            project_and_group,
+                            **configuration.get("files|" + file + "|jinja_env", dict()),
+                        )
+
+                    try:
+                        # Returns base64 encoded content: https://python-gitlab.readthedocs.io/en/stable/gl_objects/projects.html#project-files
+                        repo_file: ProjectFile = project.files.get(file_path=file, ref=branch.name)
+                        decoded_file: bytes = repo_file.decode()
+                        current_content: str = decoded_file.decode("utf-8")
+
+                        if current_content != new_content:
+                            if configuration.get("files|" + file + "|overwrite"):
+                                debug(
+                                    "Changing file '%s' in branch '%s'",
+                                    file,
+                                    branch.name,
+                                )
+                                self.modify_file_dealing_with_branch_protection(
+                                    project,
+                                    branch,
+                                    repo_file,
+                                    "modify",
+                                    configuration,
+                                    new_content,
+                                )
+                            else:
+                                debug(
+                                    "Not changing file '%s' in branch '%s' - overwrite flag not set.",
+                                    file,
+                                    branch.name,
+                                )
+                        else:
+                            debug(
+                                "Not changing file '%s' in branch '%s' - it's content is already" " as provided)",
+                                file,
+                                branch.name,
+                            )
+                    except GitlabGetError:
+                        debug("Creating file '%s' in branch '%s'", file, branch.name)
+                        self.modify_file_dealing_with_branch_protection(
+                            project,
+                            branch,
+                            file,
+                            "add",
+                            configuration,
+                            new_content,
+                        )
+
+                if configuration.get("files|" + file + "|only_first_branch"):
+                    info("Skipping other branches for this file, as configured.")
+                    break
+
+    def modify_file_dealing_with_branch_protection(
+        self,
+        project: Project,
+        branch: RESTObject,
+        file_to_operate_on: str | ProjectFile,
+        operation: str,
+        configuration: dict,
+        new_content=None,
+    ):
+        # perhaps your user permissions are ok to just perform this operation regardless
+        # of the branch protection...
+
+        try:
+            self.just_modify_file(
+                project,
+                branch,
+                file_to_operate_on,
+                operation,
+                configuration,
+                new_content,
+            )
+
+        except GitlabUpdateError as e:
+            if (
+                e.response_code == 400 or e.response_code == 403
+            ) and "You are not allowed to push into this branch" in e.error_message:
+                # If the project is archived, modifying files is not allowed
+                if project.archived:
+                    critical(f"Project is archived, cannot modify files in it.: {e.error_message}")
+                    sys.exit(EXIT_PROCESSING_ERROR)
+
+                # Otherwise, unprotect the branch but only if we know how to protect it again
+                if configuration.get("branches|" + branch.name + "|protected"):
+                    debug(f"> Temporarily unprotecting the branch to '{operation}' a file in it...")
+                    # Delete operation on protected branch removes the protection only
+                    project.protectedbranches.delete(branch.name)
+                else:
+                    critical(
+                        f"Operation '{operation}' on file in branch {branch.name} not permitted."
+                        f" We don't have a branch protection configuration provided for this"
+                        f" branch. Breaking as we cannot unprotect the branch as we would not know"
+                        f" how to protect it again."
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+                try:
+                    debug("> Attempt updating file again")
+                    self.just_modify_file(
+                        project,
+                        branch,
+                        file_to_operate_on,
+                        operation,
+                        configuration,
+                        new_content,
+                    )
+
+                finally:
+                    # ...and protect the branch again after the operation
+                    if configuration.get("branches|" + branch.name + "|protected"):
+                        debug("> Protecting the branch again.")
+                        branch_config: dict = configuration["branches"][branch.name]
+                        self.branch_processor.protect_branch(project, branch.name, branch_config)
+
+            else:
+                raise e
+
+    def just_modify_file(
+        self,
+        project: Project,
+        branch: RESTObject,
+        file_to_operate_on: str | ProjectFile,
+        operation: str,
+        configuration: dict,
+        new_content=None,
+    ):
+        if operation == "modify" and isinstance(file_to_operate_on, ProjectFile):
+            file_to_operate_on.content = new_content
+            file_to_operate_on.save(
+                commit_message=self.get_commit_message_for_file_change(
+                    "change", file_to_operate_on.file_path, configuration
+                ),
+                branch=branch.name,
+            )
+        elif operation == "add" and isinstance(file_to_operate_on, str):
+            project.files.create(
+                {
+                    "file_path": file_to_operate_on,
+                    "branch": branch.name,
+                    "content": new_content,
+                    "commit_message": self.get_commit_message_for_file_change(
+                        "delete", file_to_operate_on, configuration
+                    ),
+                }
+            )
+        elif operation == "delete" and isinstance(file_to_operate_on, ProjectFile):
+            file_to_operate_on.delete(
+                commit_message=self.get_commit_message_for_file_change(
+                    "delete", file_to_operate_on.file_path, configuration
+                ),
+                branch=branch.name,
+            )
+
+    def get_file_content_as_template(self, template, project_and_group, **kwargs):
+        # Use jinja with variables project and group
+        rtemplate = Environment(
+            loader=FileSystemLoader("."),
+            autoescape=True,
+            keep_trailing_newline=True,
+        ).from_string(template)
+        return rtemplate.render(
+            project=self.get_project(project_and_group),
+            group=self.get_group(project_and_group),
+            **kwargs,
+        )
+
+    @staticmethod
+    def get_commit_message_for_file_change(operation, file, configuration: dict):
+        commit_message = configuration.get(
+            "files|" + file + "|commit_message",
+            "Automated %s made by gitlabform" % operation,
+        )
+
+        # add '[skip ci]' to commit message to skip CI job, as documented at
+        # https://docs.gitlab.com/ee/ci/yaml/README.html#skipping-jobs
+        skip_build = configuration.get("files|" + file + "|skip_ci")
+        skip_build_str = " [skip ci]" if skip_build else ""
+
+        return f"{commit_message}{skip_build_str}"
+
+    @staticmethod
+    def get_group(project_and_group):
+        return re.match("(.*)/.*", project_and_group).group(1)
+
+    @staticmethod
+    def get_project(project_and_group):
+        return re.match(".*/(.*)", project_and_group).group(1)

gitlabform/processors/project/hooks_processor.py

@@ -0,0 +1,64 @@
+from logging import debug
+from typing import Dict, Any, List
+
+from gitlab.base import RESTObject, RESTObjectList
+from gitlab.v4.objects import Project
+from gitlab.v4.objects import ProjectHook
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class HooksProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("hooks", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        debug("Processing hooks...")
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+        project_hooks: list[ProjectHook] = project.hooks.list(get_all=True)
+
+        hooks_in_config: tuple[str, ...] = tuple(x for x in sorted(configuration["hooks"]) if x != "enforce")
+
+        for hook in hooks_in_config:
+            hook_in_gitlab: RESTObject | None = next((h for h in project_hooks if h.url == hook), None)
+            hook_config = {"url": hook}
+            hook_config.update(configuration["hooks"][hook])
+
+            hook_id = hook_in_gitlab.id if hook_in_gitlab else None
+
+            # Process hooks configured for deletion
+            if configuration.get("hooks|" + hook + "|delete"):
+                if hook_id:
+                    debug(f"Deleting hook '{hook}'")
+                    project.hooks.delete(hook_id)
+                    debug(f"Deleted hook '{hook}'")
+                else:
+                    debug(f"Not deleting hook '{hook}', because it doesn't exist")
+                continue
+
+            # Process new hook creation
+            if not hook_id:
+                debug(f"Creating hook '{hook}'")
+                created_hook: RESTObject = project.hooks.create(hook_config)
+                debug(f"Created hook: {created_hook}")
+                continue
+
+            # Processing existing hook updates
+            gl_hook: dict = hook_in_gitlab.asdict() if hook_in_gitlab else {}
+            if self._needs_update(gl_hook, hook_config):
+                debug(f"The hook '{hook}' config is different from what's in gitlab OR it contains a token")
+                debug(f"Updating hook '{hook}'")
+                updated_hook: Dict[str, Any] = project.hooks.update(hook_id, hook_config)
+                debug(f"Updated hook: {updated_hook}")
+            else:
+                debug(f"Hook '{hook}' remains unchanged")
+
+        # Process hook config enforcements
+        if configuration.get("hooks|enforce"):
+            for gh in project_hooks:
+                if gh.url not in hooks_in_config:
+                    debug(
+                        f"Deleting hook '{gh.url}' currently setup in the project but it is not in the configuration and enforce is enabled"
+                    )
+                    project.hooks.delete(gh.id)

gitlabform/processors/project/integrations_processor.py

@@ -0,0 +1,33 @@
+from logging import info
+
+from gitlab.exceptions import GitlabDeleteError
+from gitlab.v4.objects import Project, ProjectIntegration
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class IntegrationsProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("integrations", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        configured_integrations = configuration.get("integrations", {})
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+
+        for integration in sorted(configured_integrations):
+            gl_integration: ProjectIntegration = project.integrations.get(integration, lazy=True)
+
+            if configured_integrations[integration].get("delete"):
+                info(f"Deleting integration: {integration}")
+                try:
+                    gl_integration.delete()
+                except GitlabDeleteError as e:
+                    # If we get a 404 the integration does not exist, so we can ignore the error
+                    if e.response_code == 404:
+                        info(f"Integration {integration} does not exist, skipping deletion.")
+                    else:
+                        info(f"Failed to delete integration {integration}: {e}")
+                        raise
+            else:
+                info(f"Setting integration: {integration}")
+                project.integrations.update(integration, configured_integrations[integration])

gitlabform/processors/project/job_token_scope_processor.py

@@ -0,0 +1,216 @@
+from typing import List
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors import AbstractProcessor
+from logging import warning, info, debug
+
+from gitlab.v4.objects import Project, ProjectJobTokenScope
+
+
+class JobTokenScopeProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("job_token_scope", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        job_token_config = configuration.get("job_token_scope", {})
+        debug(f"Job Token Scope config: {job_token_config}")
+
+        project = self.gl.get_project_by_path_cached(project_and_group)
+        job_token_scope = project.job_token_scope.get()
+
+        limit_access_state_updated = self._process_limit_access_to_this_project_setting(
+            job_token_config, job_token_scope
+        )
+
+        if limit_access_state_updated:
+            # Refresh has no return but produces same result as project.job_token_scope.get()
+            # -> refreshes job_token_scope state with latest changes
+            job_token_scope.refresh()
+
+        allowlist_config = job_token_config.get("allowlist", {})
+        debug(f"configuration allowlist: {allowlist_config}")
+
+        info("Processing Job Token allowlist")
+
+        enforce = allowlist_config.get("enforce", False)
+
+        self._process_groups(job_token_scope, allowlist_config.get("groups", []), enforce)
+
+        self._process_projects(project, job_token_scope, allowlist_config.get("projects", []), enforce)
+
+    @staticmethod
+    def _process_limit_access_to_this_project_setting(configuration: dict, job_token_scope: ProjectJobTokenScope):
+        limit_access_to_this_project: bool = configuration.get("limit_access_to_this_project", True)
+
+        if limit_access_to_this_project != job_token_scope.inbound_enabled:
+            info(f"Updating project job token scope to limit access: {limit_access_to_this_project}")
+            job_token_scope.enabled = limit_access_to_this_project
+            job_token_scope.save()
+            return True
+        else:
+            info(f"Job Token Scope does not need updating")
+            return False
+
+    def _process_projects(
+        self,
+        project: Project,
+        job_token_scope: ProjectJobTokenScope,
+        projects_allowlist: List,
+        enforce: bool,
+    ):
+        if not projects_allowlist and enforce:
+            warning("Process will remove existing projects from allowlist, as none set in configuration")
+
+        existing_allowlist = job_token_scope.allowlist.list(get_all=True)
+
+        project_ids_to_allow = self._get_target_project_ids_from_config(projects_allowlist)
+
+        allowlist_updated = False
+        if len(project_ids_to_allow) > 0:
+            allowlist_updated = self._add_projects_to_allowlist(
+                project, job_token_scope, existing_allowlist, project_ids_to_allow
+            )
+
+        if enforce:
+            if allowlist_updated:
+                # Refresh has no return but produces same result as project.job_token_scope.get()
+                # -> refreshes job_token_scope state with latest changes
+                job_token_scope.refresh()
+
+            info("Enforce enabled, removing projects no longer defined in config from allowlist")
+            self._remove_projects_from_allowlist(project, job_token_scope, existing_allowlist, project_ids_to_allow)
+
+    def _process_groups(
+        self,
+        job_token_scope: ProjectJobTokenScope,
+        groups_allowlist: List,
+        enforce: bool,
+    ):
+        if not groups_allowlist and enforce:
+            warning("Process will remove existing groups from allowlist, as none set in configuration")
+
+        existing_allowlist = job_token_scope.groups_allowlist.list(get_all=True)
+
+        group_ids_to_allow = self._get_target_group_ids_from_config(groups_allowlist)
+
+        allowlist_updated = False
+        if len(group_ids_to_allow) > 0:
+            allowlist_updated = self._add_groups_to_allowlist(job_token_scope, existing_allowlist, group_ids_to_allow)
+
+        if enforce:
+            if allowlist_updated:
+                # Refresh has no return but produces same result as project.job_token_scope.get()
+                # -> refreshes job_token_scope state with latest changes
+                job_token_scope.refresh()
+
+            info("Enforce enabled, removing groups no longer defined in config from allowlist")
+            self._remove_groups_from_allowlist(job_token_scope, existing_allowlist, group_ids_to_allow)
+
+    def _remove_groups_from_allowlist(
+        self,
+        job_token_scope: ProjectJobTokenScope,
+        existing_allowlist,
+        target_group_ids: List,
+    ):
+        allowlist_updated = False
+        group_ids_to_remove = self._get_ids_to_remove_from_allowlist(existing_allowlist, target_group_ids)
+        for group_id in group_ids_to_remove:
+            allowlist_updated = True
+            job_token_scope.groups_allowlist.delete(group_id)
+            info("Deleted group %s from allowlist", group_id)
+
+        if allowlist_updated:
+            debug("Saving removed Groups allowlist changes")
+            job_token_scope.save()
+
+    @staticmethod
+    def _add_groups_to_allowlist(job_token_scope, existing_allowlist, group_ids_listed_in_config):
+        allowlist_updated = False
+
+        for group_id in group_ids_listed_in_config:
+            if any(allowed.get_id() == group_id for allowed in existing_allowlist):
+                # If already in allowlist, do nothing
+                debug(f"{group_id} already in Groups allowlist")
+                continue
+
+            allowlist_updated = True
+            job_token_scope.groups_allowlist.create({"target_group_id": group_id})
+            info(f"Added Group {group_id} to allowlist")
+
+        # If we have added something new to the allowlist then save the scope otherwise save API calls
+        if allowlist_updated:
+            debug("Saving added Groups allowlist changes")
+            job_token_scope.save()
+            return True
+        else:
+            return False
+
+    @staticmethod
+    def _add_projects_to_allowlist(project, job_token_scope, existing_allowlist, project_ids_listed_in_config):
+        allowlist_state_updated = False
+
+        for project_id in project_ids_listed_in_config:
+            if project_id != project.id:
+                if any(allowed.get_id() == project_id for allowed in existing_allowlist):
+                    # If already in allowlist, do nothing
+                    debug(f"{project_id} already in Projects allowlist")
+                    continue
+
+                allowlist_state_updated = True
+                job_token_scope.allowlist.create({"target_project_id": project_id})
+                info(f"Added Project {project_id} to allowlist")
+
+        # If we have added something new to the allowlist then save the scope otherwise save API calls
+        if allowlist_state_updated:
+            debug("Saving added Projects allowlist changes")
+            job_token_scope.save()
+            return True
+        else:
+            return False
+
+    def _remove_projects_from_allowlist(
+        self,
+        project: Project,
+        job_token_scope: ProjectJobTokenScope,
+        existing_allowlist,
+        target_project_ids: List,
+    ):
+        removed_items_from_allowlist = False
+        project_ids_to_remove = self._get_ids_to_remove_from_allowlist(existing_allowlist, target_project_ids)
+        for project_id in project_ids_to_remove:
+            if project_id != project.id:
+                removed_items_from_allowlist = True
+                job_token_scope.allowlist.delete(project_id)
+                info("Deleted project %s from allowlist", project_id)
+
+        if removed_items_from_allowlist:
+            debug("Saving removed Projects allowlist changes")
+            job_token_scope.save()
+
+    @staticmethod
+    def _get_ids_to_remove_from_allowlist(existing_allowlist, target_ids: List):
+        ids_to_remove = []
+
+        for allowed in existing_allowlist:
+            if allowed.id not in target_ids:
+                ids_to_remove.append(allowed.id)
+
+        return ids_to_remove
+
+    def _get_target_project_ids_from_config(self, projects_allowlist: List):
+        target_project_ids = []
+
+        for target_project_or_id in projects_allowlist:
+            target_project = self.gl.get_project_by_path_cached(target_project_or_id)
+            target_project_ids.append(target_project.id)
+
+        return target_project_ids
+
+    def _get_target_group_ids_from_config(self, groups_allowlist: List):
+        target_group_ids = []
+
+        for target_group_or_id in groups_allowlist:
+            target_group = self.gl.get_group_by_path_cached(target_group_or_id)
+            target_group_ids.append(target_group.id)
+
+        return target_group_ids