gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/processors/abstract_processor.py

@@ -0,0 +1,187 @@
+from abc import ABC, abstractmethod
+from logging import debug
+from typing import Callable, Union
+
+import requests
+from logging import info
+
+from gitlabform.gitlab import GitLab, PythonGitlab
+from gitlabform.gitlab import GitlabWrapper
+from gitlabform.output import EffectiveConfigurationFile
+from gitlabform.processors.util.decorators import configuration_to_safe_dict
+
+
+class AbstractProcessor(ABC):
+    def __init__(self, configuration_name: str, gitlab: GitLab):
+        self.configuration_name = configuration_name
+        self.gitlab = gitlab
+        self.custom_diff_analyzers: dict[
+            str,
+            Callable[[str, list[dict[str, Union[str, int]]], list[dict[str, int]]], bool],
+        ] = {}
+        self.gl: PythonGitlab = GitlabWrapper(self.gitlab).get_gitlab()
+
+    @configuration_to_safe_dict
+    def process(
+        self,
+        project_or_project_and_group: str,
+        configuration,
+        dry_run: bool,
+        diff_only_changed: bool,
+        effective_configuration: EffectiveConfigurationFile,
+    ):
+        if self._section_is_in_config(configuration):
+            if configuration.get(f"{self.configuration_name}|skip"):
+                info(f"Skipping section '{self.configuration_name}' - explicitly configured to do so.")
+                return
+            elif configuration.get("project|archive") and self.configuration_name != "project":
+                info(f"Skipping section '{self.configuration_name}' - it is configured to be archived.")
+                return
+
+            if dry_run:
+                info(f"Processing section '{self.configuration_name}' in dry-run mode.")
+                project_transfer_source = ""
+                try:
+                    project_transfer_source = configuration["project"]["transfer_from"]
+                    info(f"""Project {project_or_project_and_group} is configured to be transferred, 
+                        diffing config from transfer source project {project_transfer_source}.""")
+                except KeyError:
+                    pass
+
+                self._print_diff(
+                    project_transfer_source or project_or_project_and_group,
+                    configuration.get(self.configuration_name),
+                    diff_only_changed=diff_only_changed,
+                )
+            else:
+                info(f"Processing section '{self.configuration_name}'")
+                if self._can_proceed(project_or_project_and_group, configuration):
+                    self._process_configuration_with_retries(project_or_project_and_group, configuration)
+
+            effective_configuration.add_configuration(
+                project_or_project_and_group,
+                self.configuration_name,
+                configuration.get(self.configuration_name),
+            )
+        else:
+            info(f"Skipping section '{self.configuration_name}' - not in config.")
+
+    def _section_is_in_config(self, configuration: dict):
+        return self.configuration_name in configuration
+
+    def _process_configuration_with_retries(self, project_or_project_and_group: str, configuration: dict):
+        retry = 1
+        max_retries = 3
+
+        while True:
+            try:
+                if retry > 1:
+                    info(f"Retrying section '{self.configuration_name}' - {retry}/{max_retries}...")
+
+                self._process_configuration(
+                    project_or_project_and_group,
+                    configuration,
+                )
+
+                return
+            except Exception as e:
+                if retry > max_retries:
+                    raise MaxProcessorRetriesExceeded from e
+
+                if self._should_retry_processor(e):
+                    retry += 1
+                    continue
+                else:
+                    raise e
+
+    @staticmethod
+    def _should_retry_processor(e: Exception) -> bool:
+        # Most possible failures during processing are handled by the HTTP request retries in GitLabCore class,
+        # but in some cases we cannot do that on that level.
+
+        # If we already retried on a request level, don't retry again
+        if "Max retries exceeded with url" in str(e):
+            return False
+
+        # One case is when a POST request is made and the request is sent, but we got no (or incomplete?) response.
+        # Because we don't know if the particular POST request was done under the hood (f.e. an entity was created),
+        # we cannot retry just the single request (f.e. if it was created then a retry would either create a duplicate
+        # of the entity or fail with an error, if duplicates are not allowed in a given case).
+
+        # Then we need to retry the whole section (f.e. files) for a given entity (f.e. project foo/bar), so the checks
+        # for the initial state will re-run (f.e. checking if that entity already exists or not).
+
+        # fmt: off
+        if type(e) == requests.exceptions.ConnectionError \
+                and "RemoteDisconnected('Remote end closed connection without response')" in str(e):
+            return True
+        # fmt: on
+
+        return False
+
+    @abstractmethod
+    def _process_configuration(self, project_or_project_and_group: str, configuration: dict):
+        pass
+
+    def _print_diff(self, project_or_project_and_group: str, entity_config, diff_only_changed: bool):
+        info(f"Diffing for section '{self.configuration_name}' is not supported yet")
+
+    def _needs_update(
+        self,
+        entity_in_gitlab: dict,
+        entity_in_configuration: dict,
+    ):
+        # in the configuration we often don't define every key value because we rely on the defaults.
+        # that's why GitLab API often returns many more keys than we have in the configuration.
+
+        # so to decide if the entity should be updated:
+        # a) we look for ANY settings that are ONLY in configuration,
+        # a) we compare the settings that are in both configuration and gitlab,
+
+        keys_only_in_configuration = set(entity_in_configuration.keys()) - set(entity_in_gitlab.keys())
+        if len(keys_only_in_configuration) > 0:
+            return True
+
+        keys_on_both_sides = set(entity_in_configuration.keys()) & set(entity_in_gitlab.keys())
+        for key in keys_on_both_sides:
+            if key in self.custom_diff_analyzers:
+                return self.custom_diff_analyzers[key](key, entity_in_gitlab[key], entity_in_configuration[key])
+
+            if entity_in_gitlab[key] != entity_in_configuration[key]:
+                debug(
+                    f"entity_in_gitlab[{key}] -> {entity_in_gitlab[key]} != entity_in_configuration[{key}] -> {entity_in_configuration[key]}"
+                )
+                return True
+
+        return False
+
+    @staticmethod
+    def recursive_diff_analyzer(cfg_key: str, cfg_in_gitlab: list, local_cfg: list):
+        """
+        :return: True if the lists are NOT equal, False otherwise
+        """
+        if len(cfg_in_gitlab) != len(local_cfg):
+            return True
+
+        for index in range(len(cfg_in_gitlab)):
+            from_gitlab = {k: v for k, v in cfg_in_gitlab[index].items() if v is not None}
+            from_local_cfg = local_cfg[index]
+
+            keys_on_both_sides = set(from_gitlab.keys()) & set(from_local_cfg.keys())
+
+            for key in keys_on_both_sides:
+                if isinstance(from_gitlab[key], list) and isinstance(from_local_cfg[key], list):
+                    AbstractProcessor.recursive_diff_analyzer(key, from_gitlab[key], from_local_cfg[key])
+
+                if from_gitlab[key] != from_local_cfg[key]:
+                    debug(f"* A <{key}> in [{cfg_key}] differs:\n GitLab :: {from_gitlab} != Local :: {from_local_cfg}")
+                    return True
+
+        return False
+
+    def _can_proceed(self, project_or_group: str, configuration: dict):
+        return True
+
+
+class MaxProcessorRetriesExceeded(Exception):
+    pass

gitlabform/processors/application/__init__.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from gitlabform.configuration import Configuration
+from gitlabform.gitlab import GitLab
+from gitlabform.processors import AbstractProcessors
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.application.application_settings_processor import (
+    ApplicationSettingsProcessor,
+)
+
+
+class ApplicationProcessors(AbstractProcessors):
+    def __init__(self, gitlab: GitLab, config: Configuration, strict: bool):
+        super().__init__(gitlab, config, strict)
+        self.processors: List[AbstractProcessor] = [
+            ApplicationSettingsProcessor(gitlab),
+        ]

gitlabform/processors/application/application_settings_processor.py

@@ -0,0 +1,39 @@
+from logging import info, debug
+
+from gitlab.v4.objects import ApplicationSettings
+
+from gitlabform import GitLab
+from gitlabform.processors import AbstractProcessor
+
+
+# https://docs.gitlab.com/ee/api/settings.html
+class ApplicationSettingsProcessor(AbstractProcessor):
+
+    def __init__(self, gitlab: GitLab):
+        super().__init__("settings", gitlab)
+
+    def _process_configuration(self, project_or_group_name: str, application_configuration: dict):
+        application_settings_config = application_configuration["settings"]
+        application_settings: ApplicationSettings = self.gl.settings.get()
+
+        if self._needs_update(application_settings.asdict(), application_settings_config):
+            info("Updating Application Settings")
+            self.update_application_settings(application_settings, application_settings_config)
+        else:
+            debug("No update needed for Application Settings")
+
+    @staticmethod
+    def update_application_settings(application_settings: ApplicationSettings, application_settings_config: dict):
+        # application settings has to be like this:
+        # {
+        #     'setting1': value1,
+        #     'setting2': value2,
+        # }
+        # ..as documented at: https://docs.gitlab.com/ee/api/settings.html#change-application-settings
+
+        for key in application_settings_config:
+            value = application_settings_config[key]
+            debug(f"Updating setting {key} to value {value}")
+            application_settings.__setattr__(key, value)
+
+        application_settings.save()

gitlabform/processors/defining_keys.py

@@ -0,0 +1,152 @@
+from abc import ABC, abstractmethod
+
+
+class AbstractKey(ABC):
+    """
+    This represents a key in an entity configuration.
+
+    For example in such an entity:
+
+    rule:
+      name: foo
+      rule_type: bar
+
+    ...the "name" and "rule_type" are keys.
+
+    This may also be an expression made with keys and relationships between them, see below.
+    """
+
+    @abstractmethod
+    def matches(self, e1, e2):
+        """
+        :param e1: some entity
+        :param e2: another entity
+        :return: True if the two given entities have the same value of this key.
+        """
+        pass
+
+    @abstractmethod
+    def contains(self, entity):
+        """
+        :param entity: some entity
+        :return: True if the entity contains this key
+        """
+        pass
+
+    @abstractmethod
+    def explain(self) -> str:
+        """
+        :return: a user-friendly explanation of what this key is
+        """
+        pass
+
+
+class Key(AbstractKey):
+    """
+    A single, mandatory key.
+    """
+
+    def __init__(self, name):
+        self.name = name
+
+    def matches(self, e1, e2):
+        return self.name in e1 and self.name in e2 and e1[self.name] == e2[self.name]
+
+    def contains(self, entity):
+        return entity.get(self.name, None) is not None
+
+    def explain(self) -> str:
+        return f"'{self.name}'"
+
+
+class And(AbstractKey):
+    """
+    This groups two or more keys of which all are mandatory.
+    """
+
+    def __init__(self, *arg: AbstractKey):
+        self.keys = arg
+
+    def matches(self, e1, e2):
+        return all([key.matches(e1, e2) for key in self.keys])
+
+    def contains(self, entity):
+        return all([key.contains(entity) for key in self.keys])
+
+    def explain(self) -> str:
+        explains = [key.explain() for key in self.keys]
+        return f"({' and '.join(explains)})"
+
+
+class Or(AbstractKey):
+    """
+    This groups two or more keys where at least one must exist.
+    """
+
+    def __init__(self, *arg: AbstractKey):
+        self.keys = arg
+
+    def matches(self, e1, e2):
+        return any([key.matches(e1, e2) for key in self.keys])
+
+    def contains(self, entity):
+        return any([key.contains(entity) for key in self.keys])
+
+    def explain(self) -> str:
+        explains = [key.explain() for key in self.keys]
+        return f"({' or '.join(explains)})"
+
+
+class Xor(AbstractKey):
+    """
+    This groups two or more keys where exactly one must exist.
+    """
+
+    def __init__(self, *arg: AbstractKey):
+        self.keys = arg
+
+    # copied from https://stackoverflow.com/a/16801336/2693875
+    @staticmethod
+    def _single_true(iterable):
+        iterator = iter(iterable)
+
+        # consume from "i" until first true, or it's exhausted
+        has_true = any(iterator)
+
+        # carry on consuming until another true value / exhausted
+        has_another_true = any(iterator)
+
+        # True if exactly one true found
+        return has_true and not has_another_true
+
+    def matches(self, e1, e2):
+        return self._single_true([key.matches(e1, e2) for key in self.keys])
+
+    def contains(self, entity):
+        return self._single_true([key.contains(entity) for key in self.keys])
+
+    def explain(self) -> str:
+        explains = [key.explain() for key in self.keys]
+        return f"(exactly one of: {', '.join(explains)})"
+
+
+class OptionalKey(AbstractKey):
+    """
+    This is a non-mandatory key.
+    """
+
+    def __init__(self, name):
+        self.name = name
+
+    def matches(self, e1, e2):
+        only_in_e1 = self.name in e1 and self.name not in e2
+        only_in_e2 = self.name not in e1 and self.name in e2
+        in_both_and_equal = self.name in e1 and self.name in e2 and e1[self.name] == e2[self.name]
+        return only_in_e1 or only_in_e2 or in_both_and_equal
+
+    def contains(self, entity):
+        # as it's optional, we don't check for its existence
+        return True
+
+    def explain(self) -> str:
+        return f"(optionally '{self.name}')"

gitlabform/processors/group/__init__.py

@@ -0,0 +1,48 @@
+from typing import List
+
+from gitlabform.configuration import Configuration
+from gitlabform.gitlab import GitLab
+from gitlabform.processors import AbstractProcessors
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.group.group_badges_processor import GroupBadgesProcessor
+from gitlabform.processors.group.group_ldap_links_processor import (
+    GroupLDAPLinksProcessor,
+)
+from gitlabform.processors.group.group_members_processor import (
+    GroupMembersProcessor,
+)
+from gitlabform.processors.group.group_saml_links_processor import (
+    GroupSAMLLinksProcessor,
+)
+from gitlabform.processors.group.group_variables_processor import (
+    GroupVariablesProcessor,
+)
+from gitlabform.processors.group.group_settings_processor import (
+    GroupSettingsProcessor,
+)
+from gitlabform.processors.group.group_labels_processor import (
+    GroupLabelsProcessor,
+)
+from gitlabform.processors.group.group_push_rules_processor import (
+    GroupPushRulesProcessor,
+)
+
+from gitlabform.processors.group.group_hooks_processor import (
+    GroupHooksProcessor,
+)
+
+
+class GroupProcessors(AbstractProcessors):
+    def __init__(self, gitlab: GitLab, config: Configuration, strict: bool):
+        super().__init__(gitlab, config, strict)
+        self.processors: List[AbstractProcessor] = [
+            GroupVariablesProcessor(gitlab),
+            GroupSettingsProcessor(gitlab),
+            GroupMembersProcessor(gitlab),
+            GroupLDAPLinksProcessor(gitlab),
+            GroupBadgesProcessor(gitlab),
+            GroupSAMLLinksProcessor(gitlab),
+            GroupLabelsProcessor(gitlab),
+            GroupHooksProcessor(gitlab),
+            GroupPushRulesProcessor(gitlab),
+        ]

gitlabform/processors/group/group_badges_processor.py

@@ -0,0 +1,17 @@
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.defining_keys import Key, And
+from gitlabform.processors.multiple_entities_processor import MultipleEntitiesProcessor
+
+
+class GroupBadgesProcessor(MultipleEntitiesProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__(
+            "group_badges",
+            gitlab,
+            list_method_name="get_group_badges",
+            add_method_name="add_group_badge",
+            delete_method_name="delete_group_badge",
+            defining=Key("name"),
+            required_to_create_or_update=And(Key("name"), Key("link_url"), Key("image_url")),
+            edit_method_name="edit_group_badge",
+        )

gitlabform/processors/group/group_hooks_processor.py

@@ -0,0 +1,75 @@
+from logging import debug, warning
+from typing import Dict, Any, List
+
+from gitlab.base import RESTObject, RESTObjectList
+from gitlab.v4.objects import Group
+from gitlab.v4.objects import GroupHook
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class GroupHooksProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("group_hooks", gitlab)
+
+    def _can_proceed(self, project_or_group: str, configuration: dict):
+        if not self.gitlab.enterprise:
+            hooks_in_config = configuration["group_hooks"]
+            if hooks_in_config is not None and len(hooks_in_config) > 0:
+                # Only raise error if user has defined hooks in config, otherwise exit silently out of processor
+                warning("GitLab Community Edition does not support Group Webhooks")
+
+            return False
+
+        return True
+
+    def _process_configuration(self, group_path_and_name: str, configuration: dict):
+        hooks_in_config: tuple[str, ...] = tuple(x for x in sorted(configuration["group_hooks"]) if x != "enforce")
+
+        debug("Processing group hooks...")
+        group: Group = self.gl.get_group_by_path_cached(group_path_and_name)
+        group_hooks: list[GroupHook] = group.hooks.list(get_all=True)
+
+        for hook in hooks_in_config:
+            hook_in_gitlab: RESTObject | None = next((h for h in group_hooks if h.url == hook), None)
+            hook_config = {"url": hook}
+            hook_config.update(configuration["group_hooks"][hook])
+
+            hook_id = hook_in_gitlab.id if hook_in_gitlab else None
+
+            # Process hooks configured for deletion
+            if configuration.get("group_hooks|" + hook + "|delete"):
+                if hook_id:
+                    debug(f"Deleting group hook '{hook}'")
+                    group.hooks.delete(hook_id)
+                    debug(f"Deleted group hook '{hook}'")
+                else:
+                    debug(f"Not deleting group hook '{hook}', because it doesn't exist")
+                continue
+
+            # Process new hook creation
+            if not hook_id:
+                debug(f"Creating group hook '{hook}'")
+                created_hook: RESTObject = group.hooks.create(hook_config)
+                debug(f"Created group hook: {created_hook}")
+                continue
+
+            # Processing existing hook updates
+            gl_hook: dict = hook_in_gitlab.asdict() if hook_in_gitlab else {}
+            if self._needs_update(gl_hook, hook_config):
+                debug(f"The group hook '{hook}' config is different from what's in gitlab OR it contains a token")
+                debug(f"Updating group hook '{hook}'")
+                updated_hook: Dict[str, Any] = group.hooks.update(hook_id, hook_config)
+                debug(f"Updated group hook: {updated_hook}")
+            else:
+                debug(f"Group hook '{hook}' remains unchanged")
+
+        # Process hook config enforcements
+        if configuration.get("group_hooks|enforce"):
+            for gh in group_hooks:
+                if gh.url not in hooks_in_config:
+                    debug(
+                        f"Deleting group hook '{gh.url}' currently setup in the group but it is not in the configuration and enforce is enabled"
+                    )
+                    group.hooks.delete(gh.id)

gitlabform/processors/group/group_labels_processor.py

@@ -0,0 +1,28 @@
+from logging import info
+from typing import Dict, List
+
+
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.abstract_processor import AbstractProcessor
+from gitlabform.processors.util.labels_processor import LabelsProcessor
+
+from gitlab.v4.objects import Group
+
+
+class GroupLabelsProcessor(AbstractProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__("group_labels", gitlab)
+        self._labels_processor = LabelsProcessor()
+
+    def _process_configuration(self, group_path_and_name: str, configuration: Dict):
+        configured_labels = configuration.get("group_labels", {})
+
+        enforce = configuration.get("group_labels|enforce", False)
+
+        # Remove 'enforce' key from the config so that it's not treated as a "label"
+        if enforce:
+            configured_labels.pop("enforce")
+
+        group: Group = self.gl.get_group_by_path_cached(group_path_and_name)
+
+        self._labels_processor.process_labels(configured_labels, enforce, group, self._needs_update)

gitlabform/processors/group/group_ldap_links_processor.py

@@ -0,0 +1,16 @@
+from gitlabform.gitlab import GitLab
+from gitlabform.processors.defining_keys import And, Or, Key, Xor
+from gitlabform.processors.multiple_entities_processor import MultipleEntitiesProcessor
+
+
+class GroupLDAPLinksProcessor(MultipleEntitiesProcessor):
+    def __init__(self, gitlab: GitLab):
+        super().__init__(
+            "group_ldap_links",
+            gitlab,
+            list_method_name="get_ldap_group_links",
+            add_method_name="add_ldap_group_link",
+            delete_method_name="delete_ldap_group_link",
+            defining=And(Key("provider"), Or(Key("cn"), Key("filter"))),
+            required_to_create_or_update=And(Key("provider"), Xor(Key("cn"), Key("filter"))),
+        )