gitlabform 4.0.0__tar.gz → 4.0.2__tar.gz

{gitlabform-4.0.0 → gitlabform-4.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabform
-Version: 4.0.0
+Version: 4.0.2
 Summary: 🏗 Specialized configuration as a code tool for GitLab projects, groups and more using hierarchical configuration written in YAML
 Home-page: https://gitlabform.github.io/gitlabform
 Author: Greg Dubicki and Contributors
@@ -24,7 +24,7 @@ Requires-Dist: Jinja2==3.1.4
 Requires-Dist: luddite==1.0.4
 Requires-Dist: MarkupSafe==3.0.2
 Requires-Dist: mergedeep==1.3.4
-Requires-Dist: packaging==24.1
+Requires-Dist: packaging==24.2
 Requires-Dist: python-gitlab==5.0.0
 Requires-Dist: requests==2.32.3
 Requires-Dist: ruamel.yaml==0.17.21

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform/gitlab/__init__.py

@@ -2,7 +2,6 @@ import enum
 
 from typing import List
 
-from gitlabform.gitlab.branches import GitLabBranches
 from gitlabform.gitlab.commits import GitLabCommits
 from gitlabform.gitlab.group_badges import GitLabGroupBadges
 from gitlabform.gitlab.group_ldap_links import GitLabGroupLDAPLinks
@@ -19,7 +18,6 @@ from gitlabform.gitlab.project_merge_requests_approvals import (
 )
 from gitlabform.gitlab.python_gitlab import PythonGitlab
 from gitlabform.gitlab.variables import GitLabVariables
-from gitlabform.gitlab.repositories import GitLabRepositories
 from gitlabform.gitlab.users import GitLabUsers
 
 
@@ -49,10 +47,8 @@ class AccessLevel(enum.IntEnum):
 
 
 class GitLab(
-    GitLabBranches,
     GitLabCommits,
     GitLabMergeRequests,
-    GitLabRepositories,
     GitLabGroupLDAPLinks,
     GitLabGroupBadges,
     GitLabGroupVariables,

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform/processors/multiple_entities_processor.py

@@ -4,7 +4,7 @@ from cli_ui import fatal
 
 import abc
 from typing import Callable, Union, Any
-
+import time
 from gitlabform.constants import EXIT_INVALID_INPUT
 from gitlabform.gitlab import GitLab
 from gitlabform.processors.abstract_processor import AbstractProcessor
@@ -70,6 +70,21 @@ class MultipleEntitiesProcessor(AbstractProcessor, metaclass=abc.ABCMeta):
         else:
             enforce = False
 
+        if (
+            "members" in configuration
+            and ("protected_environments" or "merge_requests_approval_rules")
+            in configuration
+        ):
+            # When gitlabform needs to update project membership and also configure
+            # settings that are dependent on the members (i.e. environment protection
+            # or MR approval rule), there seems to be an issue in GitLab. Automated
+            # acceptance tests in gitlabform creates new user and adds to the project
+            # followed by configuring other setting. In that scenario need to wait a little
+            # before calling GitLab's REST API for processing other settings. Otherwise
+            # the users are not available to be configured in other settings and it
+            # does not return any error either"
+            time.sleep(2)
+
         # TODO: move/convert this to a configuration validation phase
         self._find_duplicates(project_or_group, entities_in_configuration)
 

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform/processors/project/__init__.py

@@ -55,8 +55,15 @@ class ProjectProcessors(AbstractProcessors):
     def __init__(self, gitlab: GitLab, config: Configuration, strict: bool):
         super().__init__(gitlab, config, strict)
         self.processors: List[AbstractProcessor] = [
+            # Order of processors matter. GitLabForm will process config sections
+            # in the order listed below. Settings that are related to each other,
+            # should be ordered accordingly. For example, branch protection or MR
+            # approvals can configure specific users, but the user must be a
+            # member of the project. So, project membership must be processed
+            # before those processors.
             ProjectProcessor(gitlab),
             ProjectSettingsProcessor(gitlab),
+            MembersProcessor(gitlab),
             ProjectPushRulesProcessor(gitlab),
             ProjectLabelsProcessor(gitlab),
             JobTokenScopeProcessor(gitlab),
@@ -67,7 +74,6 @@ class ProjectProcessors(AbstractProcessors):
             IntegrationsProcessor(gitlab),
             FilesProcessor(gitlab, config, strict),
             HooksProcessor(gitlab),
-            MembersProcessor(gitlab),
             SchedulesProcessor(gitlab),
             BadgesProcessor(gitlab),
             ResourceGroupsProcessor(gitlab),

gitlabform-4.0.2/gitlabform/processors/project/branches_processor.py

@@ -0,0 +1,151 @@
+from typing import Optional
+from logging import debug, info
+from cli_ui import warning, fatal
+from gitlab.v4.objects import Project, ProjectBranch, ProjectProtectedBranch
+from gitlab import (
+    GitlabGetError,
+    GitlabDeleteError,
+    GitlabCreateError,
+)
+from gitlabform.constants import EXIT_INVALID_INPUT, EXIT_PROCESSING_ERROR
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+import time
+
+
+class BranchesProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab, strict: bool):
+        super().__init__("branches", gitlab)
+        self.strict = strict
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        if "members" in configuration:
+            # When gitlabform needs to update project membership and also
+            # configure branch protection, there seems to be a race condition
+            # or delay in GitLab. Automated acceptance tests in gitlabform
+            # creates new user and adds to the project followed by configuring
+            # branch protection setting. In that scenario need to wait a little
+            # before calling GitLab's REST API for branch protection. Otherwise
+            # the API returns error code 422 with an message like "Push access
+            # levels user is not a member of the project"
+
+            time.sleep(2)
+
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+
+        for branch in sorted(configuration["branches"]):
+            branch_configuration: dict = self.transform_branch_config(
+                configuration["branches"][branch]
+            )
+
+            self.process_branch_protection(project, branch, branch_configuration)
+
+    def process_branch_protection(
+        self, project: Project, branch_name: str, branch_config: dict
+    ):
+        """
+        Process branch protection according to gitlabform config.
+        """
+        branch: Optional[ProjectBranch] = None
+        protected_branch: Optional[ProjectProtectedBranch] = None
+
+        try:
+            branch = project.branches.get(branch_name)
+        except GitlabGetError:
+            message = f"Branch '{branch_name}' not found when trying to processing it to be protected/unprotected!"
+            if self.strict:
+                fatal(
+                    message,
+                    exit_code=EXIT_INVALID_INPUT,
+                )
+            else:
+                warning(message)
+
+        try:
+            protected_branch = project.protectedbranches.get(branch_name)
+        except GitlabGetError:
+            message = f"The branch '{branch_name}' is not protected!"
+            debug(message)
+
+        if branch and branch_config.get("protected"):
+            if protected_branch and self._needs_update(
+                protected_branch.attributes, branch_config
+            ):
+                # Need to unprotect the branch and then re-protect again using current config
+                # Another option would be to "update" protected branch. But, GitLab's API for
+                # updating protected branch requires retrieving id of the existing rule/config
+                # such as 'allowed_to_push'. See: https://docs.gitlab.com/ee/api/protected_branches.html#example-update-a-push_access_level-record
+                # This could involve more data processing. The data for updating vs creating a
+                # protected branch is not the same. So, removing existing branch protection and
+                # reconfiguring branch protection seems simpler.
+                self.unprotect_branch(protected_branch)
+                self.protect_branch(project, branch, branch_config)
+            elif not protected_branch:
+                self.protect_branch(project, branch, branch_config)
+        elif protected_branch and not branch_config.get("protected"):
+            self.unprotect_branch(protected_branch)
+
+    def protect_branch(
+        self, project: Project, branch: ProjectBranch, branch_config: dict
+    ):
+        """
+        Protect a branch using given config.
+        Raise exception if running in strict mode.
+        """
+
+        debug("Setting branch '%s' as protected", branch.name)
+
+        try:
+            project.protectedbranches.create({"name": branch.name, **branch_config})
+        except GitlabCreateError as e:
+            message = (
+                f"Protecting branch '{branch.name}' failed! Error '{e.error_message}"
+            )
+
+            if self.strict:
+                fatal(
+                    message,
+                    exit_code=EXIT_PROCESSING_ERROR,
+                )
+            else:
+                warning(message)
+
+    def unprotect_branch(self, protected_branch: ProjectProtectedBranch):
+        """
+        Unprotect a branch.
+        Raise exception if running in strict mode.
+        """
+
+        debug("Setting branch '%s' as unprotected", protected_branch.name)
+
+        try:
+            # The delete method doesn't delete the actual branch.
+            # It only unprotects the branch.
+            protected_branch.delete()
+        except GitlabDeleteError as e:
+            message = f"Branch '{protected_branch.name}' could not be unprotected! Error '{e.error_message}"
+            if self.strict:
+                fatal(
+                    message,
+                    exit_code=EXIT_PROCESSING_ERROR,
+                )
+            else:
+                warning(message)
+
+    def transform_branch_config(self, branch_config: dict):
+        """
+        The branch configuration in gitlabform supports passing users or group using username
+        or group name but GitLab only supports their id. This method will transform the
+        config by replacing them with ids.
+        """
+
+        for key in branch_config:
+            if isinstance(branch_config[key], list):
+                for item in branch_config[key]:
+                    if isinstance(item, dict):
+                        if "user" in item:
+                            item["user_id"] = self.gl.get_user_id(item.pop("user"))
+                        elif "group" in item:
+                            item["group_id"] = self.gl.get_group_id(item.pop("group"))
+
+        return branch_config

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform/processors/project/files_processor.py

@@ -5,15 +5,18 @@ from pathlib import Path
 from logging import debug
 from cli_ui import debug as verbose
 from cli_ui import warning, fatal
+from typing import List
 
 from jinja2 import Environment, FileSystemLoader
+from gitlab import GitlabGetError, GitlabUpdateError
+from gitlab.v4.objects import Project, ProjectFile, ProjectBranch
+from gitlab.base import RESTObject
 
 from gitlabform.constants import EXIT_INVALID_INPUT
 from gitlabform.configuration import Configuration
 from gitlabform.gitlab import GitLab
-from gitlabform.gitlab.core import NotFoundException, UnexpectedResponseException
 from gitlabform.processors.abstract_processor import AbstractProcessor
-from gitlabform.processors.util.branch_protector import BranchProtector
+from gitlabform.processors.project.branches_processor import BranchesProcessor
 
 
 class FilesProcessor(AbstractProcessor):
@@ -21,32 +24,38 @@ class FilesProcessor(AbstractProcessor):
         super().__init__("files", gitlab)
         self.config = config
         self.strict = strict
-        self.branch_protector = BranchProtector(gitlab, strict)
+        self.branch_processor = BranchesProcessor(gitlab, strict)
 
     def _process_configuration(self, project_and_group: str, configuration: dict):
         for file in sorted(configuration["files"]):
+            project: Project = self.gl.get_project_by_path_cached(project_and_group)
             debug("Processing file '%s'...", file)
 
             if configuration.get("files|" + file + "|skip"):
                 debug("Skipping file '%s'", file)
                 continue
 
-            if configuration["files"][file]["branches"] == "all":
-                all_branches = self.gitlab.get_branches(project_and_group)
-                branches = sorted(all_branches)
-            elif configuration["files"][file]["branches"] == "protected":
-                protected_branches = self.gitlab.get_protected_branches(
-                    project_and_group
-                )
-                branches = sorted(protected_branches)
-            else:
-                all_branches = self.gitlab.get_branches(project_and_group)
-                branches = []
-                for branch in configuration["files"][file]["branches"]:
-                    if branch in all_branches:
-                        branches.append(branch)
-                    else:
-                        message = f"! Branch '{branch}' not found, not processing file '{file}' in it"
+            config_target_ref = configuration["files"][file]["branches"]
+            branches_to_update: List[RESTObject] = []
+
+            if isinstance(config_target_ref, str):
+                # Target ref could be either 'all' or 'protected'.
+                # Get a list of branches that should be updated.
+                if config_target_ref == "all":
+                    branches_to_update.extend(
+                        project.branches.list(get_all=True, lazy=True)
+                    )
+                elif config_target_ref == "protected":
+                    branches_to_update.extend(
+                        project.protectedbranches.list(get_all=True, lazy=True)
+                    )
+            elif isinstance(config_target_ref, list):
+                # Get a list of branches from the config that should be updated.
+                for branch_name in config_target_ref:
+                    try:
+                        branches_to_update.append(project.branches.get(branch_name))
+                    except GitlabGetError:
+                        message = f"! Branch '{branch_name}' not found, not processing file '{file}' in it"
                         if self.strict:
                             fatal(
                                 message,
@@ -55,8 +64,14 @@ class FilesProcessor(AbstractProcessor):
                         else:
                             warning(message)
 
-            for branch in branches:
-                verbose(f"Processing file '{file}' in branch '{branch}'")
+            debug(
+                "File '%s' to be updated in '%s' branche(s)",
+                file,
+                len(branches_to_update),
+            )
+
+            for branch in branches_to_update:
+                verbose(f"Processing file '{file}' in branch '{branch.name}'")
 
                 if configuration.get(
                     "files|" + file + "|content"
@@ -69,20 +84,22 @@ class FilesProcessor(AbstractProcessor):
 
                 if configuration.get("files|" + file + "|delete"):
                     try:
-                        self.gitlab.get_file(project_and_group, branch, file)
-                        debug("Deleting file '%s' in branch '%s'", file, branch)
+                        file_to_delete: ProjectFile = project.files.get(
+                            file_path=file, ref=branch.name
+                        )
+                        debug("Deleting file '%s' in branch '%s'", file, branch.name)
                         self.modify_file_dealing_with_branch_protection(
-                            project_and_group,
+                            project,
                             branch,
-                            file,
+                            file_to_delete,
                             "delete",
                             configuration,
                         )
-                    except NotFoundException:
+                    except GitlabGetError:
                         debug(
                             "Not deleting file '%s' in branch '%s' (already doesn't exist)",
                             file,
-                            branch,
+                            branch.name,
                         )
                 else:
                     # change or create file
@@ -118,16 +135,22 @@ class FilesProcessor(AbstractProcessor):
                         )
 
                     try:
-                        current_content = self.gitlab.get_file(
-                            project_and_group, branch, file
+                        repo_file: ProjectFile = project.files.get(
+                            file_path=file, ref=branch.name
                         )
+                        current_content = repo_file.content
+
                         if current_content != new_content:
                             if configuration.get("files|" + file + "|overwrite"):
-                                debug("Changing file '%s' in branch '%s'", file, branch)
+                                debug(
+                                    "Changing file '%s' in branch '%s'",
+                                    file,
+                                    branch.name,
+                                )
                                 self.modify_file_dealing_with_branch_protection(
-                                    project_and_group,
+                                    project,
                                     branch,
-                                    file,
+                                    repo_file,
                                     "modify",
                                     configuration,
                                     new_content,
@@ -136,19 +159,19 @@ class FilesProcessor(AbstractProcessor):
                                 debug(
                                     "Not changing file '%s' in branch '%s' - overwrite flag not set.",
                                     file,
-                                    branch,
+                                    branch.name,
                                 )
                         else:
                             debug(
                                 "Not changing file '%s' in branch '%s' - it's content is already"
                                 " as provided)",
                                 file,
-                                branch,
+                                branch.name,
                             )
-                    except NotFoundException:
-                        debug("Creating file '%s' in branch '%s'", file, branch)
+                    except GitlabGetError:
+                        debug("Creating file '%s' in branch '%s'", file, branch.name)
                         self.modify_file_dealing_with_branch_protection(
-                            project_and_group,
+                            project,
                             branch,
                             file,
                             "add",
@@ -162,11 +185,11 @@ class FilesProcessor(AbstractProcessor):
 
     def modify_file_dealing_with_branch_protection(
         self,
-        project_and_group,
-        branch,
-        file,
-        operation,
-        configuration,
+        project: Project,
+        branch: RESTObject,
+        file_to_operate_on: str | ProjectFile,
+        operation: str,
+        configuration: dict,
         new_content=None,
     ):
         # perhaps your user permissions are ok to just perform this operation regardless
@@ -174,36 +197,43 @@ class FilesProcessor(AbstractProcessor):
 
         try:
             self.just_modify_file(
-                project_and_group, branch, file, operation, configuration, new_content
+                project,
+                branch,
+                file_to_operate_on,
+                operation,
+                configuration,
+                new_content,
             )
 
-        except UnexpectedResponseException as e:
+        except GitlabUpdateError as e:
             if (
-                e.response_status_code == 400
-                and "You are not allowed to push into this branch" in e.response_text
+                e.response_code == 400
+                and "You are not allowed to push into this branch" in e.error_message
             ):
                 # ...but if not, then we can unprotect the branch, but only if we know how to
                 # protect it again...
 
-                if configuration.get("branches|" + branch + "|protected"):
+                if configuration.get("branches|" + branch.name + "|protected"):
                     debug(
-                        f"> Temporarily unprotecting the branch to {operation} a file in it..."
+                        f"> Temporarily unprotecting the branch to '{operation}' a file in it..."
                     )
-                    self.branch_protector.unprotect_branch(project_and_group, branch)
+                    # Delete operation on protected branch removes the protection only
+                    project.protectedbranches.delete(branch.name)
                 else:
                     fatal(
-                        f"Operation {operation} on file {file} in branch {branch} not permitted,"
-                        f" but we don't have a branch protection configuration provided for this"
+                        f"Operation '{operation}' on file in branch {branch.name} not permitted."
+                        f" We don't have a branch protection configuration provided for this"
                         f" branch. Breaking as we cannot unprotect the branch as we would not know"
                         f" how to protect it again.",
                         EXIT_INVALID_INPUT,
                     )
 
                 try:
+                    debug("> Attempt updating file again")
                     self.just_modify_file(
-                        project_and_group,
+                        project,
                         branch,
-                        file,
+                        file_to_operate_on,
                         operation,
                         configuration,
                         new_content,
@@ -211,10 +241,14 @@ class FilesProcessor(AbstractProcessor):
 
                 finally:
                     # ...and protect the branch again after the operation
-                    if configuration.get("branches|" + branch + "|protected"):
+                    if configuration.get("branches|" + branch.name + "|protected"):
                         debug("> Protecting the branch again.")
-                        self.branch_protector.protect_branch(
-                            project_and_group, configuration, branch
+                        project_branch: ProjectBranch = project.branches.get(
+                            branch.name
+                        )
+                        branch_config: dict = configuration["branches"][branch.name]
+                        self.branch_processor.protect_branch(
+                            project, project_branch, branch_config
                         )
 
             else:
@@ -222,35 +256,38 @@ class FilesProcessor(AbstractProcessor):
 
     def just_modify_file(
         self,
-        project_and_group,
-        branch,
-        file,
-        operation,
-        configuration,
+        project: Project,
+        branch: RESTObject,
+        file_to_operate_on: str | ProjectFile,
+        operation: str,
+        configuration: dict,
         new_content=None,
     ):
-        if operation == "modify":
-            self.gitlab.set_file(
-                project_and_group,
-                branch,
-                file,
-                new_content,
-                self.get_commit_message_for_file_change("change", file, configuration),
+        if operation == "modify" and isinstance(file_to_operate_on, ProjectFile):
+            file_to_operate_on.content = new_content
+            file_to_operate_on.save(
+                commit_message=self.get_commit_message_for_file_change(
+                    "change", file_to_operate_on.file_path, configuration
+                ),
+                branch=branch.name,
             )
-        elif operation == "add":
-            self.gitlab.add_file(
-                project_and_group,
-                branch,
-                file,
-                new_content,
-                self.get_commit_message_for_file_change("add", file, configuration),
+        elif operation == "add" and isinstance(file_to_operate_on, str):
+            project.files.create(
+                {
+                    "file_path": file_to_operate_on,
+                    "branch": branch.name,
+                    "content": new_content,
+                    "commit_message": self.get_commit_message_for_file_change(
+                        "delete", file_to_operate_on, configuration
+                    ),
+                }
             )
-        elif operation == "delete":
-            self.gitlab.delete_file(
-                project_and_group,
-                branch,
-                file,
-                self.get_commit_message_for_file_change("delete", file, configuration),
+        elif operation == "delete" and isinstance(file_to_operate_on, ProjectFile):
+            file_to_operate_on.delete(
+                commit_message=self.get_commit_message_for_file_change(
+                    "delete", file_to_operate_on.file_path, configuration
+                ),
+                branch=branch.name,
             )
 
     def get_file_content_as_template(self, template, project_and_group, **kwargs):

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabform
-Version: 4.0.0
+Version: 4.0.2
 Summary: 🏗 Specialized configuration as a code tool for GitLab projects, groups and more using hierarchical configuration written in YAML
 Home-page: https://gitlabform.github.io/gitlabform
 Author: Greg Dubicki and Contributors
@@ -24,7 +24,7 @@ Requires-Dist: Jinja2==3.1.4
 Requires-Dist: luddite==1.0.4
 Requires-Dist: MarkupSafe==3.0.2
 Requires-Dist: mergedeep==1.3.4
-Requires-Dist: packaging==24.1
+Requires-Dist: packaging==24.2
 Requires-Dist: python-gitlab==5.0.0
 Requires-Dist: requests==2.32.3
 Requires-Dist: ruamel.yaml==0.17.21

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform.egg-info/SOURCES.txt

@@ -20,7 +20,6 @@ gitlabform/configuration/groups.py
 gitlabform/configuration/projects.py
 gitlabform/configuration/transform.py
 gitlabform/gitlab/__init__.py
-gitlabform/gitlab/branches.py
 gitlabform/gitlab/commits.py
 gitlabform/gitlab/core.py
 gitlabform/gitlab/group_badges.py
@@ -35,7 +34,6 @@ gitlabform/gitlab/project_merge_requests_approvals.py
 gitlabform/gitlab/project_protected_environments.py
 gitlabform/gitlab/projects.py
 gitlabform/gitlab/python_gitlab.py
-gitlabform/gitlab/repositories.py
 gitlabform/gitlab/users.py
 gitlabform/gitlab/variables.py
 gitlabform/lists/__init__.py
@@ -79,7 +77,6 @@ gitlabform/processors/project/variables_processor.py
 gitlabform/processors/shared/__init__.py
 gitlabform/processors/shared/protected_environments_processor.py
 gitlabform/processors/util/__init__.py
-gitlabform/processors/util/branch_protector.py
 gitlabform/processors/util/decorators.py
 gitlabform/processors/util/difference_logger.py
 gitlabform/processors/util/labels_processor.py
@@ -134,6 +131,5 @@ tests/unit/configuration/test_subgroups.py
 tests/unit/configuration/test_yaml_version.py
 tests/unit/processors/__init__.py
 tests/unit/processors/test_abstract_processor.py
-tests/unit/processors/test_branch_protector.py
 tests/unit/processors/test_difference_logger.py
 tests/unit/processors/test_schedules_processor_extended_cron_pattern.py

{gitlabform-4.0.0 → gitlabform-4.0.2}/gitlabform.egg-info/requires.txt

@@ -5,7 +5,7 @@ Jinja2==3.1.4
 luddite==1.0.4
 MarkupSafe==3.0.2
 mergedeep==1.3.4
-packaging==24.1
+packaging==24.2
 python-gitlab==5.0.0
 requests==2.32.3
 ruamel.yaml==0.17.21

{gitlabform-4.0.0 → gitlabform-4.0.2}/setup.py

@@ -47,7 +47,7 @@ setup(
         "luddite==1.0.4",
         "MarkupSafe==3.0.2",
         "mergedeep==1.3.4",
-        "packaging==24.1",
+        "packaging==24.2",
         "python-gitlab==5.0.0",
         "requests==2.32.3",
         "ruamel.yaml==0.17.21",