gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/processors/project/schedules_processor.py

@@ -0,0 +1,208 @@
+import random
+import re
+from logging import debug
+from typing import Dict, List
+
+from gitlab.base import RESTObjectList, RESTObject
+from gitlab.v4.objects import Project, ProjectPipelineSchedule
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class SchedulesProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("schedules", gitlab)
+
+    def _process_configuration(self, project_and_group: str, configuration: Dict):
+        configured_schedules = configuration.get("schedules", {})
+
+        enforce_schedules = configuration.get("schedules|enforce", False)
+
+        # Remove 'enforce' key from the config so that it's not treated as a "schedule"
+        if enforce_schedules:
+            configured_schedules.pop("enforce")
+
+        project: Project = self.gl.get_project_by_path_cached(project_and_group)
+        existing_schedules: list[ProjectPipelineSchedule] = project.pipelineschedules.list(get_all=True)
+
+        schedule_ids_by_description: Dict = self._group_schedule_ids_by_description(existing_schedules)
+
+        for schedule_description in sorted(configured_schedules):
+            schedule_ids = schedule_ids_by_description.get(schedule_description)
+
+            if configured_schedules[schedule_description].get("delete"):
+                if schedule_ids:
+                    debug("Deleting pipeline schedules '%s'", schedule_description)
+                    for schedule_id in schedule_ids:
+                        project.pipelineschedules.get(schedule_id).delete()
+                else:
+                    debug(
+                        "Not deleting pipeline schedules '%s', because none exist",
+                        schedule_description,
+                    )
+            else:
+                entity_config = configured_schedules[schedule_description]
+
+                if schedule_ids and len(schedule_ids) == 1:
+                    schedule = project.pipelineschedules.get(schedule_ids[0])
+                    self._update_existing_schedule(entity_config, project, schedule, schedule_description)
+                elif schedule_ids:
+                    debug(
+                        "Replacing existing pipeline schedules '%s'",
+                        schedule_description,
+                    )
+                    for schedule_id in schedule_ids:
+                        project.pipelineschedules.get(schedule_id).delete()
+
+                    self._create_schedule_with_variables(entity_config, project, schedule_description)
+                else:
+                    debug("Creating pipeline schedule '%s'", schedule_description)
+                    self._create_schedule_with_variables(entity_config, project, schedule_description)
+
+        if enforce_schedules:
+            debug("Delete unconfigured schedules because enforce is enabled")
+
+            self._delete_schedules_no_longer_in_config(configured_schedules, existing_schedules, project)
+
+    def _update_existing_schedule(
+        self,
+        entity_config: Dict,
+        project: Project,
+        schedule_in_gitlab: ProjectPipelineSchedule,
+        schedule_description: str,
+    ):
+        entity_config["cron"] = _replace_extended_cron_pattern(project.id, entity_config["cron"])
+        if self._needs_update(schedule_in_gitlab.asdict(), entity_config):
+            debug("Changing existing pipeline schedule '%s'", schedule_description)
+            # In order to edit a Schedule created by someone else we need to take ownership:
+            # https://docs.gitlab.com/ee/ci/pipelines/schedules.html#take-ownership
+            shedule = project.pipelineschedules.get(schedule_in_gitlab.id)
+            shedule.take_ownership()
+            project.pipelineschedules.update(
+                schedule_in_gitlab.id,
+                {"description": schedule_description, **entity_config},
+            )
+            configured_variables = entity_config.get("variables")
+            if configured_variables is not None:
+                self._set_schedule_variables(
+                    schedule_in_gitlab,
+                    configured_variables,
+                )
+        else:
+            debug("No update required for pipeline schedule '%s'", schedule_description)
+
+    def _create_schedule_with_variables(
+        self,
+        entity_config: Dict,
+        project: Project,
+        schedule_description: str,
+    ):
+        entity_config["cron"] = _replace_extended_cron_pattern(project.id, entity_config["cron"])
+        schedule_data = {"description": schedule_description, **entity_config}
+        debug("Creating pipeline schedule using data: '%s'", schedule_data)
+        created_schedule_id = project.pipelineschedules.create(schedule_data).id
+        created_schedule = project.pipelineschedules.get(created_schedule_id)
+
+        configured_variables = entity_config.get("variables")
+        if configured_variables is not None:
+            self._set_schedule_variables(
+                created_schedule,
+                configured_variables,
+            )
+
+        created_schedule.save()
+
+    @staticmethod
+    def _set_schedule_variables(schedule: ProjectPipelineSchedule, variables: Dict) -> None:
+        attributes = schedule.attributes
+        existing_variables = attributes.get("variables")
+        if existing_variables:
+            debug("Deleting variables for pipeline schedule '%s'", schedule.description)
+
+            for variable in existing_variables:
+                schedule.variables.delete(variable.get("key"))
+
+        if variables:
+            for variable_key, variable_data in variables.items():
+                schedule.variables.create({"key": variable_key, **variable_data})
+
+    @staticmethod
+    def _group_schedule_ids_by_description(
+        schedules,
+    ) -> Dict[str, List[str]]:
+        schedule_ids_by_description: Dict[str, List[str]] = {}
+
+        for schedule in schedules:
+            schedule_ids_by_description.setdefault(schedule.description, []).append(schedule.id)
+
+        return schedule_ids_by_description
+
+    @staticmethod
+    def _delete_schedules_no_longer_in_config(configured_schedules: Dict, existing_schedules, project: Project) -> None:
+        schedule: ProjectPipelineSchedule
+        for schedule in existing_schedules:
+            schedule_description = schedule.description
+            schedule_id = schedule.id
+
+            debug(f"processing {schedule_id}: {schedule_description}")
+            if schedule_description not in configured_schedules:
+                debug(
+                    "Deleting pipeline schedule named '%s', because it is not in gitlabform configuration",
+                    schedule_description,
+                )
+                project.pipelineschedules.get(schedule_id).delete()
+
+
+class ExtendedCronPattern:
+
+    def __init__(self, project_id: int, cron_expression: str):
+        self._h_pattern = re.compile(r"H(?:(\((?P<start>\d+)-(?P<end>\d+)\))|(?P<interval>/\d+))?")
+        # We do use random here to achieve a stable pseudo-random value, this is not security relevant
+        # Seeding with project_id always returns the same numbers, that is what we want here.
+        self._random = random.Random()  # nosec B311
+        self._random.seed(project_id, 2)
+        self._cron_parts = cron_expression.split()
+        if len(self._cron_parts) != 5:
+            raise ValueError(f"Expected 5 parts in the cron expression, got {self._cron_parts}")
+
+    def render(self) -> str:
+        self._cron_parts[0] = self._detect_and_replace_h(self._cron_parts[0], 60)
+        self._cron_parts[1] = self._detect_and_replace_h(self._cron_parts[1], 24)
+        self._cron_parts[4] = self._detect_and_replace_h(self._cron_parts[4], 7)
+        return " ".join(self._cron_parts)
+
+    def _detect_and_replace_h(self, cron_part: str, default_max: int):
+        parts = cron_part.split(",")
+        for i, _ in enumerate(parts):
+            match = self._h_pattern.match(parts[i])
+            if match:
+                self._replace_h(i, parts, match, default_max)
+        return ",".join(parts)
+
+    def _replace_h(self, i: int, parts: List[str], match: re.Match, default_max: int):
+        interval = match.group("interval") or ""
+        if interval:
+            interval = int(interval[1:])
+            times = int(default_max / interval)
+            first = self._random.randint(0, interval)
+            result = [str(first)]
+            result.extend(str(first + i * interval) for i in range(1, times))
+        else:
+            start = int(match.group("start") or 0)
+            end = int(match.group("end") or default_max - 1)
+            result = [str(self._random.randint(start, end))]
+        parts[i] = parts[i].replace(match.string, ",".join(result))
+
+
+EXTENDED_CRON_PATTERN_ALIASES = {
+    "@hourly": "H * * * *",
+    "@daily": "H H * * *",
+    "@weekly": "H H * * H",
+    "@nightly": "H H(00-06) * * *",
+}
+
+
+def _replace_extended_cron_pattern(project_id: int, cron_expression: str) -> str:
+    cron_expression = EXTENDED_CRON_PATTERN_ALIASES.get(cron_expression.lower(), cron_expression)
+    return ExtendedCronPattern(project_id, cron_expression).render()

gitlabform/processors/project/tags_processor.py

@@ -0,0 +1,108 @@
+import sys
+from logging import debug, warning, critical, error
+
+from gitlabform.constants import EXIT_PROCESSING_ERROR
+from gitlabform.gitlab import GitLab
+from gitlab import GitlabDeleteError, GitlabGetError
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class TagsProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab, strict: bool):
+        super().__init__("tags", gitlab)
+        self.strict = strict
+
+    def _process_configuration(self, project_and_group: str, configuration: dict):
+        project = self.gl.get_project_by_path_cached(name=project_and_group, lazy=True)
+
+        for tag in sorted(configuration["tags"]):
+            try:
+                if configuration["tags"][tag]["protected"]:
+                    allowed_to_create = []
+
+                    if "allowed_to_create" in configuration["tags"][tag]:
+                        access_levels = set()
+                        user_ids = set()
+                        group_ids = set()
+
+                        requested_configuration = configuration["tags"][tag]["allowed_to_create"]
+
+                        for config in requested_configuration:
+                            if "access_level" in config:
+                                access_levels.add(config["access_level"])
+                            elif "user_id" in config:
+                                user_ids.add(config["user_id"])
+                            elif "user" in config:
+                                user_id = self.gl.get_user_id_cached(config["user"])
+                                if user_id is None:
+                                    error(
+                                        f"Could not find User '{config["user"]}' on the Instance, cannot Protect "
+                                        f"Tag with them"
+                                    )
+                                    raise GitlabGetError(
+                                        f"_process_configuration - No users found when searching for username {config["user"]}",
+                                        404,
+                                    )
+
+                                user_ids.add(user_id)
+                            elif "group_id" in config:
+                                group_ids.add(config["group_id"])
+                            elif "group" in config:
+                                try:
+                                    gitlab_group = self.gl.get_group_by_path_cached(config["group"])
+                                except GitlabGetError as e:
+                                    error(
+                                        f"Could not find Group '{config["group"]}' on the Instance, cannot Protect "
+                                        f"Tag with them"
+                                    )
+                                    raise e
+
+                                group_ids.add(gitlab_group.get_id())
+
+                        for val in access_levels:
+                            allowed_to_create.append({"access_level": val})
+
+                        for val in user_ids:
+                            allowed_to_create.append({"user_id": val})
+
+                        for val in group_ids:
+                            allowed_to_create.append({"group_id": val})
+
+                    create_access_level = (
+                        configuration["tags"][tag]["create_access_level"]
+                        if "create_access_level" in configuration["tags"][tag]
+                        else None
+                    )
+
+                    debug("Setting tag '%s' as *protected*", tag)
+                    try:
+                        # try to unprotect first
+                        project.protectedtags.delete(tag)
+                    except GitlabDeleteError:
+                        pass
+
+                    data = {}
+                    data["name"] = tag
+                    if allowed_to_create is not None:
+                        data["allowed_to_create"] = allowed_to_create
+                    if create_access_level is not None:
+                        data["create_access_level"] = create_access_level
+                    project.protectedtags.create(data)
+                else:
+                    debug("Setting tag '%s' as *unprotected*", tag)
+                    project.protectedtags.delete(tag)
+            except GitlabDeleteError:
+                message = f"Tag '{tag}' not found when trying to unprotect it!"
+                if self.strict:
+                    critical(message)
+                    sys.exit(EXIT_PROCESSING_ERROR)
+                else:
+                    warning(message)
+            except GitlabGetError as e:
+                if self.strict:
+                    critical(
+                        e,
+                    )
+                    sys.exit(EXIT_PROCESSING_ERROR)
+                else:
+                    warning(message)

gitlabform/processors/shared/protected_environments_processor.py

@@ -0,0 +1,20 @@
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.defining_keys import Key, And
+from gitlabform.processors.multiple_entities_processor import MultipleEntitiesProcessor
+
+
+class ProtectedEnvironmentsProcessor(MultipleEntitiesProcessor):
+    """https://docs.gitlab.com/ee/api/protected_environments.html#protect-repository-environments"""
+
+    def __init__(self, gitlab: GitLab):
+        super().__init__(
+            "protected_environments",
+            gitlab,
+            list_method_name=gitlab.list_protected_environments,
+            add_method_name=gitlab.protect_a_repository_environment,
+            delete_method_name=gitlab.unprotect_environment,
+            defining=Key("name"),
+            required_to_create_or_update=And(Key("name"), Key("deploy_access_levels")),
+        )
+
+        self.custom_diff_analyzers["deploy_access_levels"] = self.recursive_diff_analyzer

gitlabform/processors/util/decorators.py

@@ -0,0 +1,44 @@
+from functools import wraps
+
+
+class SafeDict(dict):
+    """
+    A dict that a "get" method that allows to use a path-like reference to its subdict values.
+
+    For example with a dict like {"key": {"subkey": {"subsubkey": "value"}}}
+    you can use a string 'key|subkey|subsubkey' to get the 'value'.
+
+    The default value is returned if ANY of the subelements does not exist.
+
+    Code based on https://stackoverflow.com/a/44859638/2693875
+    """
+
+    def get(self, path, default=None):
+        keys = path.split("|")
+        val = None
+
+        for key in keys:
+            if val:
+                if isinstance(val, list):
+                    val = [v.get(key, default) if v else None for v in val]
+                else:
+                    val = val.get(key, default)
+            else:
+                val = dict.get(self, key, default)
+
+            if not val:
+                break
+
+        return val
+
+
+def configuration_to_safe_dict(method):
+    """
+    This wrapper function calls the method with the configuration converted from a regular dict into a SafeDict
+    """
+
+    @wraps(method)
+    def method_wrapper(self, project_and_group, configuration, *args):
+        return method(self, project_and_group, SafeDict(configuration), *args)
+
+    return method_wrapper

gitlabform/processors/util/difference_logger.py

@@ -0,0 +1,70 @@
+import hashlib
+import json
+from itertools import starmap
+
+from logging import info
+
+
+# Simple function to create strings for values which should be hidden
+# example: <secret b2c1a982>
+def hide(text: str):
+    return f"<secret {hashlib.sha256(text.encode('utf-8')).hexdigest()[:8]}>"
+
+
+class DifferenceLogger:
+    @staticmethod
+    def log_diff(
+        subject,
+        current_config,
+        config_to_apply,
+        only_changed=False,
+        hide_entries=None,
+        test=False,
+    ):
+        # Compose values in list of `[key, from_config, from_server]``
+        changes = [
+            [
+                k,
+                json.dumps(current_config.get(k, "???") if isinstance(current_config, dict) else "???"),
+                json.dumps(v),
+            ]
+            for k, v in config_to_apply.items()
+        ]
+
+        # Remove unchanged if needed
+        if only_changed:
+            # due to `filter` returning an iterator, we have to wrap it
+            # in `list()` to get the values and assign back to `changes`,
+            # otherwise `changes` is not what we expect it to be later :)
+            changes = list(filter(lambda i: i[1] != i[2], changes))
+
+        # Hide secrets
+        if hide_entries:
+            changes = list(
+                map(
+                    lambda i: ([i[0], hide(i[1]), hide(i[2])] if i[0] in hide_entries else i),
+                    changes,
+                )
+            )
+
+        # There is the potential that no changes need to be shown, which
+        # results in the calls to max() later on to fail. Instead, opting
+        # to return early with an emtpy string, since no changes were identified
+        if len(changes) == 0:
+            return ""
+
+        # calculate field size for nice formatting
+        max_key_len = str(max(map(lambda i: len(i[0]), changes)))
+        max_val_1 = str(max(map(lambda i: len(i[1]), changes)))
+        max_val_2 = str(max(map(lambda i: len(i[2]), changes)))
+
+        # generate placeholders for output pattern: `     value: before  => after   `
+        pattern = "{:>" + max_key_len + "}: {:<" + max_val_1 + "} => {:<" + max_val_2 + "}"
+
+        # create string
+        text = "{subject}:\n{diff}".format(subject=subject, diff="\n".join(starmap(pattern.format, changes)))
+
+        if test:
+            return text
+        else:
+            info(text)

gitlabform/processors/util/labels_processor.py

@@ -0,0 +1,120 @@
+from logging import info, warning
+from typing import Dict, List, Callable, Union
+
+from gitlab.v4.objects import Group, Project, ProjectLabel, GroupLabel
+
+
+class LabelsProcessor:
+
+    # Groups and Projects share the same API for .labels within python-gitlab
+    def process_labels(
+        self,
+        configured_labels: Dict,
+        enforce: bool,
+        group_or_project: Group | Project,
+        needs_update: Callable,  # self._needs_update passed from AbstractProcessor called process_labels
+    ):
+        # Only get Labels created directly on the project/group
+        existing_group_labels = group_or_project.labels.list(get_all=True, include_ancestor_groups=False)
+        existing_group_and_parent_labels = group_or_project.labels.list(get_all=True)
+        existing_label_keys: List = []
+
+        if isinstance(group_or_project, Group):
+            parent_object_type = "Group"
+        else:
+            parent_object_type = "Project"
+
+        gitlab_labels_to_delete: List = []
+
+        for label_to_update in existing_group_labels:
+            label_name_in_gl = label_to_update.name
+            updated_label = False
+            info(f"Checking if {label_name_in_gl} is in Configuration to update or delete")
+
+            for key, configured_label in configured_labels.items():
+                configured_label_name = configured_label.get("name")
+                # Key in YAML may not match the "name" value in Gitlab or YAML, so we must match on both
+                if self.configured_label_matches_gitlab_label(configured_label_name, key, label_name_in_gl):
+                    # label exists in GL, so update
+                    existing_label_keys.append(key)
+                    updated_label = True
+
+                    if needs_update(label_to_update.asdict(), configured_label):
+                        self.update_existing_label(
+                            configured_label,
+                            self.get_label(group_or_project, label_to_update),
+                            parent_object_type,
+                        )
+                    else:
+                        info(f"No update required for label: {label_name_in_gl}")
+                    break
+
+            if not updated_label:
+                gitlab_labels_to_delete.append(label_to_update)
+
+        # Delete labels no longer in config
+        for label_to_delete in gitlab_labels_to_delete:
+            label_name_in_gl = label_to_delete.name
+
+            info(f"{label_name_in_gl} not in configured labels")
+            # only delete labels when enforce is true, because user's maybe automatically applying labels based
+            # on Repo state, for example: Compliance Framework labels based on language or CI-template status
+            if enforce:
+                info(f"Removing {label_name_in_gl} from {parent_object_type}")
+                self.get_label(group_or_project, label_to_delete).delete()
+
+        # add new labels
+
+        for label_key in configured_labels.keys():
+            if label_key not in existing_label_keys:
+                info(f"Creating new label with key: {label_key}, on {parent_object_type}")
+                self.create_new_label(
+                    configured_labels, group_or_project, label_key, parent_object_type, existing_group_and_parent_labels
+                )
+
+    @staticmethod
+    def configured_label_matches_gitlab_label(configured_label_name: str, key: str, label_name_in_gl: str):
+        return (
+            configured_label_name is not None and label_name_in_gl == configured_label_name
+        ) or label_name_in_gl == key
+
+    @staticmethod
+    def get_label(group_or_project, listed_label) -> GroupLabel | ProjectLabel:
+        return group_or_project.labels.get(listed_label.id)
+
+    @staticmethod
+    def update_existing_label(
+        configured_label,
+        full_label: GroupLabel | ProjectLabel,
+        parent_object_type: str,
+    ):
+        info(f"Updating {full_label.name} on {parent_object_type}")
+
+        # label APIs in python-gitlab do not supply an update() method
+        for key in configured_label:
+            full_label.__setattr__(key, configured_label[key])
+
+        full_label.save()
+
+    def create_new_label(
+        self,
+        configured_labels,
+        group_or_project: Group | Project,
+        label_key: str,
+        parent_object_type: str,
+        existing_group_and_parent_labels: Union[List[GroupLabel], List[ProjectLabel]],
+    ):
+        label = configured_labels.get(label_key)
+        configured_label_name = label.get("name")
+        found_existing_label = False
+        for existing_label in existing_group_and_parent_labels:
+            if self.configured_label_matches_gitlab_label(configured_label_name, label_key, existing_label.name):
+                warning(
+                    f"Label {existing_label.name} already exists either in {group_or_project.name} or on Parent Group, so will not create"
+                )
+                found_existing_label = True
+                break
+
+        if not found_existing_label:
+            info(f"Adding label with key: {label_key} to {parent_object_type}")
+            group_or_project.labels.create({"name": label_key, **label})