gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/gitlab/core.py

@@ -0,0 +1,334 @@
+import functools
+import os
+import re
+from logging import debug, info, warning
+from typing import Union
+from urllib import parse
+
+from packaging import version
+
+from importlib.metadata import version as package_version
+import requests
+
+# noinspection PyPackageRequirements
+import urllib3
+from requests.adapters import HTTPAdapter
+
+# noinspection PyPackageRequirements
+from urllib3.util.retry import Retry
+
+from gitlabform.configuration import Configuration
+from gitlabform.util import to_str
+
+
+class GitLabCore:
+    def __init__(self, config_path=None, config_string=None):
+        self.configuration = Configuration(config_path, config_string)
+
+        default_gitlab_config = {
+            "url": os.getenv("GITLAB_URL"),
+            "token": os.getenv("GITLAB_TOKEN"),
+            "ssl_verify": True,
+            "timeout": 10,
+            "max_retries": 3,
+            "backoff_factor": 0.25,
+            "retry_transient_errors": True,
+        }
+        gitlab_config_from_file = self.configuration.get("gitlab", {})
+        self.gitlab_config = {**default_gitlab_config, **gitlab_config_from_file}
+
+        self.session = requests.Session()
+
+        retries_status_forcelist = []
+        if self.gitlab_config["retry_transient_errors"]:
+            # 429 Too Many Requests is included to handle rate limiting
+            #     Ideally we would like to handle Rate Limiting retrys based on 'Retry-After' header
+            #     As done in python-gitlab: https://github.com/python-gitlab/python-gitlab/blob/main/docs/api-usage-advanced.rst#rate-limits
+            # 5xx status codes are included to retry after transient server errors
+            retries_status_forcelist = [429, 500, 502, 503, 504] + list(range(520, 531))
+
+        retries = Retry(
+            total=self.gitlab_config["max_retries"],
+            backoff_factor=self.gitlab_config["backoff_factor"],
+            status_forcelist=retries_status_forcelist,
+        )
+
+        self.session.mount("http://", HTTPAdapter(max_retries=retries))
+        self.session.mount("https://", HTTPAdapter(max_retries=retries))
+
+        self.session.verify = self.gitlab_config["ssl_verify"]
+        if not self.gitlab_config["ssl_verify"]:
+            urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+        self.gitlabform_version = package_version("gitlabform")
+        self.requests_version = package_version("requests")
+        self.session.headers.update(
+            {
+                "private-token": self.gitlab_config["token"],
+                "authorization": f"Bearer {self.gitlab_config['token']}",
+                "user-agent": f"GitLabForm/{self.gitlabform_version} (python-requests/{self.requests_version})",
+            }
+        )
+
+        try:
+            version_response = self._make_requests_to_api("version")
+            info(
+                f"Connected to GitLab version: {version_response['version']} ({version_response['revision']}), Enterprise Edition: {version_response['enterprise']}"
+            )
+            self.version = version_response["version"]
+            self.enterprise = version_response["enterprise"]
+
+            if self.is_version_less_than("16"):
+                warning(
+                    f"Support for GitLab version {self.version} is Deprecated. See Requirements: https://gitlabform.github.io/gitlabform/requirements/"
+                )
+
+            current_user = self._make_requests_to_api("user")
+            if current_user.get("is_admin", False):
+                self.admin = True
+            else:
+                self.admin = False
+            info(f"Connected as: {current_user['username']}, admin: {'yes' if self.admin else 'no'}")
+            if not self.admin:
+                warning("Connected as non-admin. You may encounter permission issues.")
+
+        except Exception as e:
+            raise TestRequestFailedException(e)
+
+    def get_configuration(self):
+        return self.configuration
+
+    def get_project(self, project_and_group_or_id):
+        return self._make_requests_to_api("projects/%s", project_and_group_or_id)
+
+    def is_version_at_least(self, min_version: Union[str, version.Version]) -> bool:
+        """
+        Check if GitLab server version is at least the specified version
+
+        Args:
+             min_version: Version string like "15.4.0" or "16.0", or a Version object
+
+        Returns:
+            bool: True if server version is >= min_version, False otherwise
+        """
+        current_version = "0.0.0"
+
+        if self.version != "unknown":
+            sem_ver_regex = "^\d*\.\d*\.\d*"
+            # Get the pure semantic version from self.version, for example Gitlab will return 18.2.1-ee
+            match = re.search(sem_ver_regex, self.version)
+            if match:
+                current_version = match.group()
+
+        if isinstance(min_version, str):
+            min_version = version.parse(min_version)
+
+        return version.parse(current_version) >= min_version
+
+    def is_version_less_than(self, max_version: Union[str, version.Version]) -> bool:
+        """
+        Check if GitLab server version is less than the specified version
+        Args:
+             max_version: Version string like "15.4.0" or "16.0", or a Version object
+
+        Returns:
+            bool: True if server version is < max_version, False otherwise
+        """
+
+        return not self.is_version_at_least(max_version)
+
+    @functools.lru_cache()
+    def _get_user_id(self, username: str) -> int:
+        users = self._make_requests_to_api("users?username=%s", username, "GET")
+
+        # this API endpoint is for lookup, not search, so 'username' has to be full and exact username
+        # also it's not possible to get more than 1 user as a result
+
+        if len(users) == 0:
+            raise NotFoundException("No users found when searching for username '%s'" % username)
+
+        return int(users[0]["id"])
+
+    @functools.lru_cache()
+    def _get_group_id(self, path) -> int:
+        group = self._make_requests_to_api("groups/%s", path, "GET")
+        return int(group["id"])
+
+    @functools.lru_cache()
+    def _get_protected_branch_id(self, project_and_group_name, branch) -> int:
+        branch = self._make_requests_to_api("projects/%s/protected_branches/%s", (project_and_group_name, branch))
+        return int(branch["id"])
+
+    @functools.lru_cache()
+    def _get_project_id(self, project_and_group):
+        # This is a NEW workaround for https://github.com/gitlabhq/gitlabhq/issues/8290
+        result = self.get_project(project_and_group)
+        return str(result["id"])
+
+    def _make_requests_to_api(
+        self,
+        path_as_format_string,
+        args=None,
+        method="GET",
+        data=None,
+        expected_codes=200,
+        json=None,
+    ):
+        """
+        Makes an HTTP request or requests to the GitLab API endpoint. More than one request is made automatically
+        if the endpoint is paginated. (See underlying method for authentication, retries, timeout etc.)
+
+        :param path_as_format_string: path with parts to be replaced by values from `args` replaced by '%s'
+                                      (aka the old-style Python string formatting, see:
+                                       https://docs.python.org/2/library/stdtypes.html#string-formatting )
+        :param args: single element or a tuple of values to put under '%s's in `path_as_format_string`
+        :param method: uppercase string of a HTTP method name, like 'GET' or 'PUT'
+        :param data: dict with data to be 'PUT'ted or 'POST'ed
+        :param expected_codes: a single HTTP code (like: 200) or a list of accepted HTTP codes
+                               - if the call to the API will return other code an exception will be thrown
+        :param json: alternatively to `dict` you can set this to a string that can be parsed as JSON that will
+                     be used as data to be 'PUT'ted or 'POST'ed
+        :return: data returned by the endpoint, as a JSON object. If the API is paginated, it returns JSONs with
+                 arrays of objects and then this method returns JSON with a single array that contains all of those
+                 objects.
+        """
+        if method != "GET":
+            response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
+            return response.json()
+        else:
+            if "?" in path_as_format_string:
+                path_as_format_string += "&per_page=100"
+            else:
+                path_as_format_string += "?per_page=100"
+
+            first_response = self._make_request_to_api(path_as_format_string, args, method, data, expected_codes, json)
+            results = first_response.json()
+
+            # In newer versions of GitLab the 'x-total-pages' may not be available
+            # anymore, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43159
+            # so let's use the 'x-next-page' header instead
+
+            response = first_response
+            while True:
+                if "x-next-page" in response.headers and response.headers["x-next-page"]:
+                    next_page = response.headers["x-next-page"]
+                    response = self._make_request_to_api(
+                        path_as_format_string + "&page=" + str(next_page),
+                        args,
+                        method,
+                        data,
+                        expected_codes,
+                        json,
+                    )
+                    results += response.json()
+                else:
+                    break
+
+        return results
+
+    def _make_request_to_api(self, path_as_format_string, args, method, dict_data, expected_codes, json_data):
+        """
+        Makes a single request to the GitLab API. Takes care of the authentication, basic error processing,
+        retries, timeout etc.
+
+        :param for the params description please see `_make_requests_to_api()`
+        :return: data returned by the endpoint, as a JSON object.
+        """
+
+        expected_codes = self._listify(expected_codes)
+
+        if dict_data and json_data:
+            raise Exception("You need to pass the data either as dict (dict_data) or JSON (json_data), not both!")
+
+        url = f"{self.gitlab_config['url']}/api/v4/{self._format_with_url_encoding(path_as_format_string, args)}"
+        if dict_data:
+            response = self.session.request(method, url, data=dict_data, timeout=self.gitlab_config["timeout"])
+            debug(f"===> data = {to_str(dict_data)}")
+        elif json_data:
+            response = self.session.request(method, url, json=json_data, timeout=self.gitlab_config["timeout"])
+            debug(f"===> json = {to_str(json_data)}")
+        else:
+            response = self.session.request(method, url, timeout=self.gitlab_config["timeout"])
+
+        if response.status_code in expected_codes:
+            # if we accept error responses then they will likely not contain a JSON body
+            # so fake it to fix further calls to response.json()
+            if response.status_code == 204 or (400 <= response.status_code <= 499):
+                response.json = lambda: {}
+        else:
+            if response.status_code == 404:
+                raise NotFoundException(f"Resource with url='{url}' not found (HTTP 404)!")
+            else:
+                if dict_data:
+                    data_output = f"data='{to_str(dict_data)}' "
+                elif json_data:
+                    data_output = f"json='{to_str(json_data)}' "
+                else:
+                    data_output = ""
+
+                raise UnexpectedResponseException(
+                    f"Request url='{url}', method={method}, {data_output}failed -"
+                    f" expected code(s) {str(expected_codes)},"
+                    f" got code {response.status_code} & body: '{response.text}'",
+                    response.status_code,
+                    response.text,
+                )
+        if response.json():
+            debug(f"<--- json = {to_str(response.json())}")
+        else:
+            debug(f"<--- json = (empty))")
+        return response
+
+    @staticmethod
+    def _format_with_url_encoding(format_string, single_arg_or_args_tuple):
+        # we want to URL-encode all the args, but not the path itself which looks like "/foo/%s/bar"
+        # because '/'s here are NOT to be URL-encoded
+
+        if not single_arg_or_args_tuple:
+            # there are no params, so the format_string is the URL
+            return format_string
+        else:
+            if type(single_arg_or_args_tuple) == tuple:
+                # URL-encode each arg in the tuple and return it as tuple too
+                url_encoded_args = ()
+                for arg in single_arg_or_args_tuple:
+                    url_encoded_args += (parse.quote_plus(str(arg)),)
+            else:
+                # URL-encode single arg
+                url_encoded_args = parse.quote_plus(str(single_arg_or_args_tuple))
+
+            return format_string % url_encoded_args
+
+    @staticmethod
+    def _listify(expected_codes):
+        if isinstance(expected_codes, int):
+            return [expected_codes]
+        else:
+            return expected_codes
+
+
+class TestRequestFailedException(Exception):
+    def __init__(self, underlying: Exception):
+        self.underlying = underlying
+
+
+class NotFoundException(Exception):
+    pass
+
+
+class TimeoutWaitingForDeletion(Exception):
+    pass
+
+
+class InvalidParametersException(Exception):
+    pass
+
+
+class UnexpectedResponseException(Exception):
+    def __init__(self, message: str, response_status_code: int, response_text: str):
+        self.message: str = message
+        self.response_status_code: int = response_status_code
+        self.response_text: str = response_text
+
+    def __str__(self):
+        return self.message

gitlabform/gitlab/group_badges.py

@@ -0,0 +1,50 @@
+from gitlabform.gitlab.groups import GitLabGroups
+
+
+class GitLabGroupBadges(GitLabGroups):
+    def get_group_badges(self, group_path):
+        # (unlike the one for project badges) this endpoint returns ONLY group badges
+        return self._make_requests_to_api(
+            "groups/%s/badges",
+            group_path,
+            expected_codes=200,
+        )
+
+    def add_group_badge(
+        self,
+        group_path,
+        badge_in_config,
+    ):
+        return self._make_requests_to_api(
+            "groups/%s/badges",
+            group_path,
+            method="POST",
+            data=badge_in_config,
+            expected_codes=201,
+        )
+
+    def edit_group_badge(
+        self,
+        group_path,
+        badge_in_gitlab,
+        badge_in_config,
+    ):
+        return self._make_requests_to_api(
+            "groups/%s/badges/%s",
+            (group_path, badge_in_gitlab["id"]),
+            method="PUT",
+            data=badge_in_config,
+        )
+
+    def delete_group_badge(
+        self,
+        group_path,
+        badge_in_gitlab,
+    ):
+        # 404 means it is already removed, so let's accept it for idempotency
+        return self._make_requests_to_api(
+            "groups/%s/badges/%s",
+            (group_path, badge_in_gitlab["id"]),
+            method="DELETE",
+            expected_codes=[200, 204, 404],
+        )

gitlabform/gitlab/group_ldap_links.py

@@ -0,0 +1,40 @@
+from gitlabform.gitlab.core import NotFoundException, InvalidParametersException
+from gitlabform.gitlab.groups import GitLabGroups
+
+
+class GitLabGroupLDAPLinks(GitLabGroups):
+    def get_ldap_group_links(self, group):
+        group_id = self.get_group_id_case_insensitive(group)
+        return self._make_requests_to_api("groups/%s/ldap_group_links", group_id, expected_codes=[200, 404])
+
+    def add_ldap_group_link(self, group, data):
+        group_id = self.get_group_id_case_insensitive(group)
+        data["id"] = group_id
+
+        try:
+            return self._make_requests_to_api(
+                "groups/%s/ldap_group_links",
+                group_id,
+                method="POST",
+                data=data,
+                expected_codes=[200, 201],
+            )
+        # this is a GitLab API bug - it returns 404 here instead of 400 for bad requests...
+        except NotFoundException:
+            raise InvalidParametersException(f"Invalid parameters for a Group LDAP link for group {group}: {data}")
+
+    def delete_ldap_group_link(self, group, data):
+        if "group_access" in data:
+            del data["group_access"]
+
+        group_id = self.get_group_id_case_insensitive(group)
+        data["id"] = group_id
+
+        # 404 means that the LDAP group link is already removed, so let's accept it for idempotency
+        self._make_requests_to_api(
+            "groups/%s/ldap_group_links",
+            group_id,
+            method="DELETE",
+            data=data,
+            expected_codes=[204, 404],
+        )

gitlabform/gitlab/groups.py

@@ -0,0 +1,96 @@
+import functools
+
+from gitlabform.gitlab.core import GitLabCore, NotFoundException
+
+
+class GitLabGroups(GitLabCore):
+    @functools.lru_cache()
+    def get_group_id_case_insensitive(self, some_string):
+        # Cache the mapping from some_string -> id, as that won't change during our run.
+        return self.get_group_case_insensitive(some_string)["id"]
+
+    def get_group_case_insensitive(self, some_string):
+        # maybe "foo/bar" is some group's path
+
+        try:
+            # try with exact case
+            return self.get_group(some_string)
+        except NotFoundException:
+            # try case insensitive
+            groups = self._make_requests_to_api(
+                "groups?search=%s",
+                some_string.lower(),
+                method="GET",
+            )
+
+            for group in groups:
+                if group["full_path"].lower() == some_string.lower():
+                    return group
+            raise NotFoundException(f"Group/subgroup with path '{some_string}' not found.")
+
+    def get_group(self, name):
+        return self._make_requests_to_api("groups/%s", name)
+
+    def get_group_descendants(self, group_id_or_path):
+        return self._make_requests_to_api("groups/%s/descendant_groups", group_id_or_path)
+
+    def get_groups(self):
+        """
+        :return: sorted list of groups
+        """
+
+        if self.admin:
+            query = "all_available=true"
+        else:
+            # as a non-admin it's pointless to get groups with a lower role than Reporter
+            # - it's the minimal role that is needed to manage something (f.e. group labels)
+            query = f"min_access_level=20"
+
+        result = self._make_requests_to_api(f"groups?{query}")
+
+        return sorted(map(lambda x: x["full_path"], result))
+
+    def get_projects(self, group, include_archived=False, only_names=True):
+        """
+        :param group: group name
+        :param include_archived: set to True if archived projects should also be returned
+        :param only_names: set to False to get the whole project objects
+        :return: sorted list of strings "group/project_name". Note that only projects from "group" namespace are
+                 returned, so if "group" (= members of this group) is also a member of some projects, they won't be
+                 returned here.
+        """
+        try:
+            # there are 3 states of the "archived" flag: true, false, undefined
+            # we use the last 2
+            if include_archived:
+                query1 = "include_subgroups=true"
+            else:
+                query1 = "include_subgroups=true&archived=false"
+
+            if self.admin:
+                query2 = "all_available=true"
+            else:
+                # it's pointless to get projects with a lower role than Reporter
+                # - it's the minimal role that is needed to manage something (f.e. labels)
+                query2 = f"min_access_level=20"
+
+            projects = self._make_requests_to_api(f"groups/%s/projects?{query1}&{query2}", group)
+        except NotFoundException:
+            projects = []
+
+        project_and_groups_in_group_namespace = [
+            project for project in projects if project["path_with_namespace"].startswith(group + "/")
+        ]
+
+        if only_names:
+            return sorted(
+                map(
+                    lambda x: x["path_with_namespace"],
+                    project_and_groups_in_group_namespace,
+                )
+            )
+        else:
+            return sorted(
+                project_and_groups_in_group_namespace,
+                key=lambda x: x["path_with_namespace"],
+            )

gitlabform/gitlab/merge_requests.py

@@ -0,0 +1,57 @@
+from gitlabform.gitlab.core import GitLabCore
+
+
+class GitLabMergeRequests(GitLabCore):
+    def create_mr(
+        self,
+        project_and_group_name,
+        source_branch,
+        target_branch,
+        title,
+        description=None,
+    ):
+        data = {
+            "id": project_and_group_name,
+            "source_branch": source_branch,
+            "target_branch": target_branch,
+            "title": title,
+            "description": description,
+        }
+        return self._make_requests_to_api(
+            "projects/%s/merge_requests",
+            project_and_group_name,
+            method="POST",
+            data=data,
+            expected_codes=201,
+        )
+
+    def accept_mr(self, project_and_group_name, mr_iid):
+        return self._make_requests_to_api(
+            "projects/%s/merge_requests/%s/merge",
+            (project_and_group_name, mr_iid),
+            method="PUT",
+        )
+
+    def update_mr(self, project_and_group_name, mr_iid, data):
+        self._make_requests_to_api(
+            "projects/%s/merge_requests/%s",
+            (project_and_group_name, mr_iid),
+            method="PUT",
+            data=data,
+        )
+
+    def get_mrs(self, project_and_group_name):
+        """
+        :param project_and_group_name: like 'group/project'
+        :return: get all *open* MRs in given project
+        """
+        return self._make_requests_to_api(
+            "projects/%s/merge_requests?scope=all&state=opened",
+            project_and_group_name,
+        )
+
+    def get_mr(self, project_and_group_name, mr_iid):
+        return self._make_requests_to_api("projects/%s/merge_requests/%s", (project_and_group_name, mr_iid))
+
+    def get_mr_approvals(self, project_and_group_name, mr_iid):
+        return self._make_requests_to_api("projects/%s/merge_requests/%s/approvals", (project_and_group_name, mr_iid))

gitlabform/gitlab/pipelines.py

@@ -0,0 +1,23 @@
+from gitlabform.gitlab.core import GitLabCore
+
+
+class GitLabPipelines(GitLabCore):
+    def get_pipelines(self, project_and_group_name, branch):
+        pipelines = self._make_requests_to_api(
+            "projects/%s/pipelines?ref=%s",
+            (project_and_group_name, branch),
+        )
+        return pipelines
+
+    def get_pipeline(self, project_and_group_name, pipeline_id):
+        pipeline = self._make_requests_to_api("/projects/%s/pipelines/%s", (project_and_group_name, pipeline_id))
+        return pipeline
+
+    def retry_pipeline(self, project_and_group_name, pipeline_id):
+        pipeline = self._make_requests_to_api(
+            "projects/%s/pipelines/%s/retry",
+            (project_and_group_name, pipeline_id),
+            method="POST",
+            expected_codes=[200, 201],
+        )
+        return pipeline

gitlabform/gitlab/project_badges.py

@@ -0,0 +1,52 @@
+from gitlabform.gitlab.projects import GitLabProjects
+
+
+class GitLabProjectBadges(GitLabProjects):
+    def get_project_badges(self, project_and_group_name):
+        badges = self._make_requests_to_api(
+            "projects/%s/badges",
+            project_and_group_name,
+            expected_codes=200,
+        )
+        # according to the docs_new this endpoint returns also the group badges
+        # but we want only the project badges here
+        return [badge for badge in badges if badge["kind"] == "project"]
+
+    def add_project_badge(
+        self,
+        project_and_group_name,
+        badge_in_config,
+    ):
+        return self._make_requests_to_api(
+            "projects/%s/badges",
+            project_and_group_name,
+            method="POST",
+            data=badge_in_config,
+            expected_codes=201,
+        )
+
+    def edit_project_badge(
+        self,
+        project_and_group_name,
+        badge_in_gitlab,
+        badge_in_config,
+    ):
+        return self._make_requests_to_api(
+            "projects/%s/badges/%s",
+            (project_and_group_name, badge_in_gitlab["id"]),
+            method="PUT",
+            data=badge_in_config,
+        )
+
+    def delete_project_badge(
+        self,
+        project_and_group_name,
+        badge_in_gitlab,
+    ):
+        # 404 means it is already removed, so let's accept it for idempotency
+        return self._make_requests_to_api(
+            "projects/%s/badges/%s",
+            (project_and_group_name, badge_in_gitlab["id"]),
+            method="DELETE",
+            expected_codes=[200, 204, 404],
+        )