gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/gitlab/project_deploy_keys.py

@@ -0,0 +1,102 @@
+from gitlabform.gitlab.core import UnexpectedResponseException
+from gitlabform.gitlab.projects import GitLabProjects
+
+
+class GitLabProjectDeployKeys(GitLabProjects):
+    def get_all_deploy_keys(self):
+        return self._make_requests_to_api("deploy_keys")
+
+    def get_deploy_keys(self, project_and_group_name):
+        return self._make_requests_to_api("projects/%s/deploy_keys", project_and_group_name)
+
+    def post_deploy_key(self, project_and_group_name, deploy_key_in_config):
+        # deploy_key_in_config has to be like this:
+        # {
+        #     'title': title,
+        #     'key': key,
+        #     'can_push': can_push,
+        # }
+        # ..as documented at: https://docs.gitlab.com/ce/api/deploy_keys.html#add-deploy-key
+
+        try:
+            self._make_requests_to_api(
+                "projects/%s/deploy_keys",
+                project_and_group_name,
+                "POST",
+                deploy_key_in_config,
+                expected_codes=[201],
+            )
+
+        except UnexpectedResponseException as e:
+            if e.response_status_code == 400 and ("has already been taken" in e.response_text):
+                # Sometimes GitLab throws HTTP 400: {"deploy_key.fingerprint_sha256":["has already been taken"]}
+                # when you try to add an existing SSH key to another project, although according to the API docs
+                # it should work
+                # (see https://docs.gitlab.com/ee/api/deploy_keys.html#add-deploy-keys-to-multiple-projects).
+
+                # As a workaround, when that happens, we will try to get this key's id and enable this existing key
+                # for the new project.
+
+                all_existing_keys = self.get_all_deploy_keys()
+
+                existing_key_id = None
+                for existing_key in all_existing_keys:
+                    if self._keys_are_effectively_equal(existing_key["key"], deploy_key_in_config["key"]):
+                        existing_key_id = existing_key["id"]
+                        break
+
+                if existing_key_id:
+                    # POST /projects/:id/deploy_keys/:key_id/enable
+                    self._make_requests_to_api(
+                        "projects/%s/deploy_keys/%s/enable",
+                        (project_and_group_name, existing_key_id),
+                        "POST",
+                        expected_codes=201,
+                    )
+                else:
+                    e.message = (
+                        e.message
+                        + "\n"
+                        + "!!! However we haven't found this key in GitLab under the /deploy_keys API... !!!"
+                    )
+                    raise e
+
+    def put_deploy_key(self, project_and_group_name, deploy_key_in_gitlab, deploy_key_in_config):
+        # according to docs_new at https://docs.gitlab.com/ee/api/deploy_keys.html#update-deploy-key
+        # you only can change key's title and can_push param
+
+        changeable_data = {
+            "title": deploy_key_in_config.get("title", None),
+            "can_push": deploy_key_in_config.get("can_push", None),
+        }
+
+        return self._make_requests_to_api(
+            "projects/%s/deploy_keys/%s",
+            (project_and_group_name, deploy_key_in_gitlab["id"]),
+            "PUT",
+            changeable_data,
+        )
+
+    def delete_deploy_key(self, project_and_group_name, deploy_key_in_config):
+        return self._make_requests_to_api(
+            "projects/%s/deploy_keys/%s",
+            (project_and_group_name, deploy_key_in_config["id"]),
+            method="DELETE",
+            expected_codes=[204, 404],
+        )
+
+    def get_deploy_key(self, project_and_group_name, id):
+        return self._make_requests_to_api("projects/%s/deploy_keys/%s", (project_and_group_name, id), "GET")
+
+    @staticmethod
+    def _keys_are_effectively_equal(key1, key2):
+        # We ignore the comment part of the SSH key as GitLab doesn't allow adding the same key just
+        # with a different comment BUT it also has a bug that it returns keys with only parts of the
+        # comments if the comment contains spaces, so it may show a difference where there is none...
+
+        key1_type = key1.split(" ")[0]
+        key1_value = key1.split(" ")[1]
+        key2_type = key2.split(" ")[0]
+        key2_value = key2.split(" ")[1]
+
+        return (key1_type == key2_type) and (key1_value == key2_value)

gitlabform/gitlab/project_merge_requests_approvals.py

@@ -0,0 +1,94 @@
+from gitlabform.gitlab.core import (
+    NotFoundException,
+    GitLabCore,
+)
+
+
+class GitLabProjectMergeRequestsApprovals(GitLabCore):
+    # approval rules
+
+    def get_approval_rules(self, project_and_group_name):
+        # for this endpoint GitLab still actually wants pid, not "group/project"...
+        pid = self._get_project_id(project_and_group_name)
+        return self._make_requests_to_api("projects/%s/approval_rules", pid)
+
+    def get_approval_rule(self, project_and_group_name, name):
+        # for this endpoint GitLab still actually wants pid, not "group/project"...
+        pid = self._get_project_id(project_and_group_name)
+        rules = self._make_requests_to_api("projects/%s/approval_rules", pid)
+        for rule in rules:
+            if rule["name"] == name:
+                return rule
+        raise NotFoundException
+
+    def add_approval_rule(
+        self,
+        project_and_group_name,
+        data,
+    ):
+        pid = self._get_project_id(project_and_group_name)
+
+        if "protected_branches" in data:
+            self._transform_protected_branches(data, project_and_group_name)
+
+        self._make_requests_to_api(
+            "projects/%s/approval_rules",
+            pid,
+            method="POST",
+            data=None,
+            expected_codes=201,
+            json=data,
+        )
+
+    def edit_approval_rule(
+        self,
+        project_and_group_name,
+        rule_in_gitlab,
+        rule_in_config,
+    ):
+        pid = self._get_project_id(project_and_group_name)
+        approval_rule_id = rule_in_gitlab["id"]
+
+        # GitLab interprets not passing any of these lists as "do not change them"
+        # while what we really what is in this case is "clear them"
+        if "user_ids" not in rule_in_config:
+            rule_in_config["user_ids"] = []
+        if "group_ids" not in rule_in_config:
+            rule_in_config["group_ids"] = []
+        if "protected_branches" in rule_in_config:
+            self._transform_protected_branches(rule_in_config, project_and_group_name)
+        else:
+            rule_in_config["protected_branch_ids"] = []
+
+        self._make_requests_to_api(
+            "projects/%s/approval_rules/%s",
+            (pid, approval_rule_id),
+            method="PUT",
+            data=None,
+            json=rule_in_config,
+        )
+
+    def delete_approval_rule(self, project_and_group_name, rule_in_gitlab):
+        # for this endpoint GitLab still actually wants pid, not "group/project"...
+        pid = self._get_project_id(project_and_group_name)
+        approval_rule_id = rule_in_gitlab["id"]
+
+        self._make_requests_to_api(
+            "projects/%s/approval_rules/%s",
+            (pid, approval_rule_id),
+            method="DELETE",
+            expected_codes=[200, 204],
+        )
+
+    def _transform_protected_branches(self, data, project_and_group_name):
+        # we do this transformation here instead of in a ConfigurationTransformers
+        # because we need a context of the project and group to convert branch names to ids
+        # and that is not available there
+
+        protected_branches_name = data["protected_branches"]
+        data.pop("protected_branches")
+        protected_branch_ids = []
+        for branch_name in protected_branches_name:
+            branch_id = self._get_protected_branch_id(project_and_group_name, branch_name)
+            protected_branch_ids.append(branch_id)
+        data["protected_branch_ids"] = protected_branch_ids

gitlabform/gitlab/project_protected_environments.py

@@ -0,0 +1,37 @@
+from logging import info
+
+from gitlabform.gitlab.projects import GitLabProjects
+
+
+class GitLabProjectProtectedEnvironments(GitLabProjects):
+    def list_protected_environments(self, project_and_group_name: str):
+        return self._make_requests_to_api("projects/%s/protected_environments", project_and_group_name)
+
+    def protect_a_repository_environment(
+        self, project_and_group_name: str, protected_env_cfg: dict, retry: bool = True
+    ):
+        response = self._make_requests_to_api(
+            "projects/%s/protected_environments",
+            project_and_group_name,
+            method="POST",
+            json=protected_env_cfg,
+            expected_codes=201,
+        )
+
+        # TODO: remove this when this issue is resolved -> https://gitlab.com/gitlab-org/gitlab/-/issues/378657
+        if retry and (len(protected_env_cfg["deploy_access_levels"]) != len(response["deploy_access_levels"])):
+            info(f'Gitlab\'s returned "deploy_access_levels" differs from the sent cfg, trying again...')
+
+            self.unprotect_environment(project_and_group_name, protected_env_cfg)
+
+            return self.protect_a_repository_environment(project_and_group_name, protected_env_cfg, False)
+
+        return response
+
+    def unprotect_environment(self, project_and_group_name: str, protected_env_cfg: dict):
+        return self._make_requests_to_api(
+            "projects/%s/protected_environments/%s",
+            (project_and_group_name, protected_env_cfg["name"]),
+            method="DELETE",
+            expected_codes=204,
+        )

gitlabform/gitlab/projects.py

@@ -0,0 +1,151 @@
+from time import sleep
+
+from gitlabform.gitlab.core import (
+    GitLabCore,
+    NotFoundException,
+    TimeoutWaitingForDeletion,
+)
+
+
+class GitLabProjects(GitLabCore):
+    def get_project_case_insensitive(self, some_string):
+        # maybe "foo/bar" is some project's path
+
+        try:
+            # try with exact case
+            return self.get_project(some_string)
+        except NotFoundException:
+            # try case insensitive
+            projects = self._make_requests_to_api(
+                f"projects?search=%s&simple=true",
+                some_string.lower(),
+                method="GET",
+            )
+            for project in projects:
+                if project["path_with_namespace"].lower() == some_string.lower():
+                    return project
+            raise NotFoundException(f"Project with path '{some_string}' not found.")
+
+    def create_project(
+        self,
+        name,
+        path,
+        namespace_id,
+        default_branch=None,
+        wait_if_still_being_deleted=False,
+    ):
+        data = {
+            "name": name,
+            "path": path,
+            "namespace_id": namespace_id,
+        }
+        if default_branch:
+            data["default_branch"] = default_branch
+
+        if wait_if_still_being_deleted:
+            # GitLab deletes the project asynchronously, it may take a few seconds.
+            # So if you are creating new project with the same name as the one
+            # that is still being deleted, GitLab returns code 400
+            # and "The project is still being deleted.". Let's retry a few times
+            # then to start creating when the deletion is done.
+            # (Note: this code DOES NOT support the "Delayed Project deletion" feature
+            # where the actual deletion can be postponed for days!)
+
+            max_retries = 10
+            wait_before_retry = 3
+            retry = 0
+
+            while True:
+                retry += 1
+
+                if retry > max_retries:
+                    raise TimeoutWaitingForDeletion
+
+                response = self._make_requests_to_api("projects", data=data, method="POST", expected_codes=[201, 400])
+                if self._is_project_still_deleted(response):
+                    # wait & retry
+                    sleep(wait_before_retry)
+                    continue
+                else:
+                    return response
+
+        else:
+            return self._make_requests_to_api("projects", data=data, method="POST", expected_codes=201)
+
+    @staticmethod
+    def _is_project_still_deleted(response):
+        # check if response looks like this:
+        # {'message': {'base': ['The project is still being deleted. Please try again later.'],
+        # 'limit_reached': []}}
+        return (
+            "message" in response
+            and "base" in response["message"]
+            and type(response["message"]["base"]) == list
+            and len(response["message"]["base"]) == 1
+            and "The project is still being deleted." in response["message"]["base"][0]
+        )
+
+    def delete_project(self, project_and_group_name):
+        # 404 means that the project does not exist anymore, so let's accept it for idempotency
+        return self._make_requests_to_api(
+            "projects/%s",
+            project_and_group_name,
+            method="DELETE",
+            expected_codes=[202, 204, 404],
+        )
+
+        # GitLab deletes the project asynchronously, it may take a few seconds
+        # BUT it doesn't return such not yet deleted project in GET calls, so
+        # there is no point in checking if it actually done here. :(
+        # See create_project() for the code that deals with that.
+
+    def get_all_projects(self, include_archived=False):
+        """
+        :param include_archived: if the archived projects should be returned too
+        :return: sorted list of ALL projects you have access to, strings like: "group/project_name"
+        """
+        try:
+            # there are 3 states of the "archived" flag: true, false, undefined
+            # we use the last 2
+            if include_archived:
+                query_string = "order_by=name&sort=asc"
+            else:
+                query_string = "order_by=name&sort=asc&archived=false"
+            result = self._make_requests_to_api(f"projects?{query_string}")
+            return sorted(map(lambda x: x["path_with_namespace"], result))
+        except NotFoundException:
+            return []
+
+    def get_groups_from_project(self, project_and_group_name):
+        # couldn't find an API call that was giving me directly
+        # the shared groups, so I'm using directly the GET /projects/:id call
+        project_info = self._make_requests_to_api("projects/%s", project_and_group_name)
+
+        # it will return {group_name: {...api info about group_name...}, ...}
+        groups = {}
+        for group in project_info["shared_with_groups"]:
+            groups[group["group_full_path"]] = group
+
+        return groups
+
+    def share_with_group(self, project_and_group_name, group_name, group_access, expires_at):
+        data = {"group_id": self._get_group_id(group_name), "expires_at": expires_at}
+        if group_access is not None:
+            data["group_access"] = group_access
+        return self._make_requests_to_api(
+            "projects/%s/share",
+            project_and_group_name,
+            method="POST",
+            data=data,
+            expected_codes=201,
+        )
+
+    def unshare_with_group(self, project_and_group_name, group_name):
+        # 404 means that the group already has not access, so let's accept it for idempotency
+        group_id = self._get_group_id(group_name)
+        return self._make_requests_to_api(
+            "projects/%s/share/%s",
+            (project_and_group_name, group_id),
+            method="DELETE",
+            expected_codes=[204, 404],
+        )

gitlabform/gitlab/python_gitlab.py

@@ -0,0 +1,251 @@
+import functools
+from typing import Union, Any, Optional, Dict, List
+
+import gitlab.const
+from gitlab import Gitlab, GitlabGetError, GraphQL
+from gitlab.base import RESTObject
+from gitlab.v4.objects import Group, Project, User
+
+from logging import info
+
+
+# Extends the python-gitlab class to add convenience wrappers for common functionality used within gitlabform
+class PythonGitlab(Gitlab):
+    def __init__(
+        self,
+        graphql: GraphQL,
+        url: Optional[str] = None,
+        private_token: Optional[str] = None,
+        oauth_token: Optional[str] = None,
+        job_token: Optional[str] = None,
+        ssl_verify: Union[bool, str] = True,
+        http_username: Optional[str] = None,
+        http_password: Optional[str] = None,
+        timeout: Optional[float] = None,
+        api_version: str = "4",
+        per_page: Optional[int] = None,
+        pagination: Optional[str] = None,
+        order_by: Optional[str] = None,
+        user_agent: str = gitlab.const.USER_AGENT,
+        retry_transient_errors: bool = False,
+        keep_base_url: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(
+            url,
+            private_token,
+            oauth_token,
+            job_token,
+            ssl_verify,
+            http_username,
+            http_password,
+            timeout,
+            api_version,
+            per_page,
+            pagination,
+            order_by,
+            user_agent,
+            retry_transient_errors,
+            keep_base_url,
+            **kwargs,
+        )
+        # Python Gitlab GraphQL interface
+        # https://python-gitlab.readthedocs.io/en/stable/api-usage-graphql.html
+        self.graphql = graphql
+
+    def get_user_id_cached(self, username) -> int | None:
+        """
+        Get user id from username
+        API call layer "get_user_by_username" is cached, we do not cache at this level as it would add
+        duplication of data into the cache
+        """
+        user = self.get_user_by_username_cached(username)
+
+        if user is None:
+            return None
+
+        return user.id
+
+    def get_group_id(self, groupname) -> int:
+        group = self.get_group_by_path_cached(groupname)
+        return group.id
+
+    def get_project_id(self, name) -> int:
+        project = self.get_project_by_path_cached(name)
+        return project.id
+
+    @functools.lru_cache()
+    def get_project_by_path_cached(self, name: str, lazy: bool = False) -> Project:
+        project: Project = self.projects.get(name, lazy)
+        if project:
+            return project
+
+        raise GitlabGetError("No project found when getting '%s'" % name, 404)
+
+    @functools.lru_cache()
+    def get_group_by_path_cached(self, groupname: str) -> Group:
+        group: Group = self.groups.get(groupname)
+        if group:
+            return group
+
+        raise GitlabGetError("No group found when getting '%s'" % groupname, 404)
+
+    #  Uses "LIST" to get a user by username, to get the full User object, call get using the user's id
+    @functools.lru_cache()
+    def get_user_by_username_cached(self, username: str) -> User | None:
+        # Gitlab API will only ever return 0 or 1 entry when GETting using `username` attribute
+        # https://docs.gitlab.com/ee/api/users.html#for-non-administrator-users
+        # so will always be list[RESTObject] and never RESTObjectList from python-gitlab's api
+        # username list is case-insensitive on the gitlab api
+        users: list[RESTObject] = self.users.list(username=username)  # type: ignore
+
+        if len(users) == 0:
+            return None
+
+        user = users[0]
+        return self.users.get(user.id)
+
+    @functools.lru_cache()
+    def _get_member_roles_from_group_cached(self, group_full_path: str) -> List[Dict[str, str]]:
+        """Query GraphQL using Python Gitlab
+        https://python-gitlab.readthedocs.io/en/stable/api-usage-graphql.html
+
+        GET Member_Role via REST Api needs "Owner" on Gitlab.com or "Admin" on Dedicated
+        List of custom roles can be retrieved via GraphQL endpoint with "Guest+"
+        https://gitlab.com/gitlab-org/gitlab/-/issues/511919#note_2287581884
+
+        https://docs.gitlab.com/ee/api/graphql/reference/index.html#groupmemberroles
+        """
+
+        if group_full_path is None:
+            raise GitlabGetError(
+                "Group Path must be provided when getting member roles",
+                404,
+            )
+
+        query = (
+            """
+        {
+          group(fullPath: \""""
+            + group_full_path
+            + """\") {
+            memberRoles {
+              nodes {
+                id
+                name
+              }
+            }
+          }
+        }
+        """
+        )
+        info(f"Executing graphQl query to get Member Roles for Group '{group_full_path}'")
+        result = self.graphql.execute(query)
+
+        # Validate Group / MemberRoles exist
+        if (
+            result["group"] is not None
+            and result["group"]["memberRoles"] is not None
+            and result["group"]["memberRoles"]["nodes"] is not None
+        ):
+            member_role_nodes = result["group"]["memberRoles"]["nodes"]
+        else:
+            raise GitlabGetError(f"Failed to get Member Roles from Group: {query}")
+
+        return self._convert_result_to_member_roles(member_role_nodes)
+
+    @functools.lru_cache()
+    def _get_member_roles_from_instance_cached(self) -> List[Dict[str, str]]:
+        """Query GraphQL using Python Gitlab
+        https://python-gitlab.readthedocs.io/en/stable/api-usage-graphql.html
+
+        GET Member_Role via REST Api needs "Admin" on Dedicated/Self-Hosted
+
+        On Self-Hosted/Dedicated it is only possible to create Member Roles at an instance level, so we query at that
+        level and cache the result in order to save API calls when processing multiple Groups
+
+        List of custom roles can be retrieved via GraphQL endpoint with "Guest+"
+        https://gitlab.com/gitlab-org/gitlab/-/issues/511919#note_2287581884
+
+        https://docs.gitlab.com/ee/api/graphql/reference/index.html#groupmemberroles
+        """
+
+        query = """
+            {
+              memberRoles {
+                edges {
+                  node {
+                    id
+                    name
+                  }
+                }
+              }
+            }
+            """
+        result = self.graphql.execute(query)
+
+        if result["memberRoles"] is not None and result["memberRoles"]["edges"] is not None:
+            member_role_edges = result["memberRoles"]["edges"]
+        else:
+            raise GitlabGetError(f"Failed to get Member Roles from instance: {query}")
+
+        member_role_nodes = []
+
+        for edge in member_role_edges:
+            member_role_nodes.append(edge["node"])
+
+        return self._convert_result_to_member_roles(member_role_nodes)
+
+    @staticmethod
+    def _convert_result_to_member_roles(member_role_nodes: List[Dict[str, str]]) -> List[Dict[str, str]]:
+        # Unwrap Ids from GraphQl Gids
+        member_role_id_prefix = "gid://gitlab/MemberRole/"
+
+        member_roles = []
+        for node in member_role_nodes:
+            member_role_id = node["id"]
+            member_role_id = member_role_id.replace(member_role_id_prefix, "")
+            member_roles.append({"id": member_role_id, "name": node["name"]})
+
+        return member_roles
+
+    @staticmethod
+    def _get_member_role_from_member_roles(
+        name_or_id: Union[int, str], member_roles: List[Dict[str, str]]
+    ) -> Dict[str, str]:
+        for member_role in member_roles:
+            member_role_id: str = member_role["id"]
+            member_role_name: str = member_role["name"]
+            if int(member_role_id) == name_or_id:
+                return member_role
+            elif member_role_name.lower() == str(name_or_id).lower():
+                return member_role
+
+        # Failed to find member role so throw an exception explaining such to user
+        raise GitlabGetError(
+            f"Member Role with name or id {name_or_id} could not be found",
+            404,
+        )
+
+    def get_member_role_id_cached(self, name_or_id: Union[int, str], group_full_path: str) -> int:
+        """
+        GETs a member role id set in the config by the Name of the Member Role (by calling out to API) or by Id
+
+        Get member_roles calls themselves are Cached, the logic in this method otherwise performs no API calls,
+        so we should not cache the result of this specific method, otherwise we may fill the cache with duplicate data
+        """
+        if type(name_or_id) is int:
+            # Already supplied as an id so no need to go get it from API
+            return name_or_id
+
+        if self._is_gitlab_saas():
+            member_roles = self._get_member_roles_from_group_cached(group_full_path)
+        else:
+            member_roles = self._get_member_roles_from_instance_cached()
+
+        member_role = self._get_member_role_from_member_roles(name_or_id, member_roles)
+
+        return int(member_role["id"])
+
+    def _is_gitlab_saas(self) -> bool:
+        return self.url == gitlab.const.DEFAULT_URL

gitlabform/gitlab/variables.py

@@ -0,0 +1,47 @@
+from gitlabform.gitlab.projects import GitLabProjects
+
+
+class GitLabVariables(GitLabProjects):
+    def get_variables(self, project_and_group_name):
+        return self._make_requests_to_api("projects/%s/variables", project_and_group_name)
+
+    def post_variable(self, project_and_group_name, variable_in_config):
+        # variable has to be like documented at:
+        # https://docs.gitlab.com/ee/api/project_level_variables.html#create-variable
+        self._make_requests_to_api(
+            "projects/%s/variables",
+            project_and_group_name,
+            "POST",
+            variable_in_config,
+            expected_codes=201,
+        )
+
+    def put_variable(
+        self,
+        project_and_group_name,
+        variable_in_gitlab,
+        variable_in_config,
+    ):
+        # variable has to be like documented at:
+        # https://docs.gitlab.com/ce/api/build_variables.html#update-variable
+        self._make_requests_to_api(
+            "projects/%s/variables/%s",
+            (project_and_group_name, variable_in_gitlab["key"]),
+            "PUT",
+            variable_in_config,
+        )
+
+    def delete_variable(self, project_and_group_name, variable_in_config):
+        self._make_requests_to_api(
+            "projects/%s/variables/%s",
+            (project_and_group_name, variable_in_config["key"]),
+            method="DELETE",
+            expected_codes=[204, 404],
+        )
+
+    def get_variable(self, project_and_group_name, variable_key, environment_scope="*"):
+        if environment_scope == "*":
+            url = "projects/%s/variables/%s"
+        else:
+            url = f"projects/%s/variables/%s?filter[environment_scope]={environment_scope}"
+        return self._make_requests_to_api(url, (project_and_group_name, variable_key))["value"]