gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/configuration/__init__.py

@@ -0,0 +1,12 @@
+from gitlabform.configuration.common import ConfigurationCommon
+from gitlabform.configuration.groups import ConfigurationGroups
+from gitlabform.configuration.projects import ConfigurationProjects
+
+#
+# This the only external interface for operating on the configuration from a given YAML file
+# (or an input string in case of tests).
+#
+
+
+class Configuration(ConfigurationProjects):
+    pass

gitlabform/configuration/common.py

@@ -0,0 +1,19 @@
+from abc import ABC
+
+from gitlabform.configuration.core import ConfigurationCore
+
+
+class ConfigurationCommon(ConfigurationCore, ABC):
+    """
+    Gets the common configuration, applied to all groups and projects.
+    """
+
+    def get_common_config(self) -> dict:
+        """
+        :return: literal common configuration or empty dict if not defined
+        """
+        common_config = self.get("projects_and_groups|*", {})
+        if common_config:
+            section_name = "*"
+            self._validate_break_inheritance_flag(common_config, section_name)
+        return common_config

gitlabform/configuration/core.py

@@ -0,0 +1,323 @@
+import sys
+from typing import Any
+
+import os
+import logging
+import textwrap
+from abc import ABC
+from copy import deepcopy
+from logging import debug
+from pathlib import Path
+from ruamel.yaml.scalarstring import ScalarString
+from types import SimpleNamespace
+
+from logging import critical, info
+
+from mergedeep import merge
+from yamlpath.common import Parsers
+from yamlpath.wrappers import ConsolePrinter
+
+from gitlabform.constants import EXIT_INVALID_INPUT
+from ruamel.yaml.comments import CommentedMap
+
+
+class ConfigurationCore(ABC):
+    """
+    The functionality shared by various types of config (common, groups, projects)
+    is implemented here.
+    """
+
+    def __init__(self, config_path=None, config_string=None):
+        if config_path and config_string:
+            critical("Please initialize with either config_path or config_string, not both.")
+            sys.exit(EXIT_INVALID_INPUT)
+        try:
+            if config_string:
+                self.config = self._parse_yaml(config_string, config_string=True)
+                self.config_dir = "."
+            else:  # maybe config_path
+                config_path = self._get_config_path(config_path)
+                self.config = self._parse_yaml(config_path, config_string=False)
+                self.config_dir = os.path.dirname(config_path)
+
+                # below checks are only needed in the non-test mode, when the config is read from file
+
+                if self.config.get("example_config"):
+                    critical(
+                        "Example config detected, aborting.\n"
+                        "Haven't you forgotten to use `-c <config_file>` parameter?\n"
+                        "If you created your config based on the example config.yml,"
+                        " then please remove 'example_config' key."
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+                if self.config.get("config_version", 1) != 4:
+                    critical(
+                        "This version of GitLabForm requires 'config_version: 4' entry in the config. "
+                        "This ensures that if the application behavior changes in a backward-incompatible way,"
+                        " you won't apply unwanted configuration to your GitLab instance.\n"
+                        "Please follow this guide: https://gitlabform.github.io/gitlabform/upgrade/\n"
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+            self._find_almost_duplicates()
+
+            # we are NOT checking for the existence of non-empty 'projects_and_groups' key here
+            # as it would break using GitLabForm as a library
+
+        except (FileNotFoundError, OSError):
+            raise ConfigFileNotFoundException(config_path)
+
+        except Exception as e:
+            raise ConfigInvalidException(e)
+
+    @staticmethod
+    def _get_config_path(config_path):
+        if "APP_HOME" in os.environ:
+            # using this env var should be considered unofficial, we need this temporarily
+            # for backwards compatibility. support for it may be removed without notice, do not use it!
+            config_path = os.path.join(os.environ["APP_HOME"], "config.yml")
+        elif not config_path:
+            # this case is only meant for using gitlabform as a library
+            config_path = os.path.join(str(Path.home()), ".gitlabform", "config.yml")
+        elif config_path in [os.path.join(".", "config.yml"), "config.yml"]:
+            # provided points to config.yml in the app current working dir
+            config_path = os.path.join(os.getcwd(), "config.yml")
+
+        return config_path
+
+    @staticmethod
+    def _parse_yaml(source: str, config_string: bool):
+        logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False)
+        log = ConsolePrinter(logging_args)
+
+        yaml = Parsers.get_yaml_editor()
+
+        # for better backward compatibility with PyYAML (that supports only YAML 1.1) used in the previous
+        # GitLabForm versions, let's force ruamel.yaml to use YAML version 1.1 by default too
+        yaml.version = (1, 1)
+
+        if config_string:
+            config = textwrap.dedent(source)
+            info("Reading config from the provided string.")
+            yaml_data, doc_loaded = Parsers.get_yaml_data(yaml, log, config, literal=True)
+        else:
+            config_path = source
+            info(f"Reading config from file: {config_path}")
+            yaml_data, doc_loaded = Parsers.get_yaml_data(yaml, log, config_path)
+
+        if doc_loaded:
+            debug("Config parsed successfully as YAML.")
+        else:
+            # an error message has already been printed via ConsolePrinter
+            exit(EXIT_INVALID_INPUT)
+
+        return yaml_data
+
+    def get(self, path, default=None) -> Any:
+        """
+        :param path: "path" to given element in YAML file, for example for:
+
+        group_settings:
+          sddc:
+            deploy_keys:
+              qa_puppet:
+                key: some key...
+                title: some title...
+                can_push: false
+
+        ..a path to a single element array ['qa_puppet'] will be: "group_settings|sddc|deploy_keys".
+
+        To get the dict under it use: get("group_settings|sddc|deploy_keys")
+
+        :param default: the value to return if the key is not found. The default 'None' means that an exception
+                        will be raised in such case.
+        :return: element from YAML file (dict, array, string...)
+        """
+        tokens = path.split("|")
+        current = self.config
+
+        try:
+            for token in tokens:
+                current = current[token]
+
+            if isinstance(current, ScalarString):
+                to_return = str(current)
+            else:
+                to_return = current
+        except:
+            if default is not None:
+                to_return = default
+            else:
+                raise KeyNotFoundException(path) from None
+
+        return to_return
+
+    @staticmethod
+    def _validate_break_inheritance_flag(config: dict, section_name: str, parent_key: str = "") -> None:
+        for key, value in config.items():
+            if "inherit" == key:
+                parent_key_description = ' under key "' + parent_key + '"' if parent_key else ""
+                critical(
+                    f'The inheritance-break flag set in "{section_name}"{parent_key_description} is invalid\n'
+                    f"because it has no higher level setting to inherit from.\n"
+                )
+                sys.exit(EXIT_INVALID_INPUT)
+            elif type(value) in [CommentedMap, dict]:
+                ConfigurationCore._validate_break_inheritance_flag(value, section_name, key)
+
+    @staticmethod
+    def _merge_configs(more_general_config, more_specific_config) -> dict:
+        """
+        :return: merge more general config with more specific configs.
+                 More specific config values take precedence over more general ones.
+        """
+
+        more_general_config = deepcopy(more_general_config)
+        more_specific_config = deepcopy(more_specific_config)
+
+        merged_dict = merge({}, more_general_config, more_specific_config)
+
+        def break_inheritance(specific_config, parent_path=()):
+            """
+            Walk the specific config tree and replace only the exact nested section
+            that declares ``inherit: false``.
+
+            ``parent_path`` stores the full key path to the current section because
+            the same section name, like ``variables``, can appear in multiple branches.
+            Replacing by section name alone can therefore update the wrong subtree.
+            """
+            for key, value in specific_config.items():
+                if "inherit" == key:
+                    if not value:
+                        replace_config_section(merged_dict, parent_path, specific_config)
+                        break
+                    elif value:
+                        critical(f"Cannot set the inheritance break flag with true\n")
+                        sys.exit(EXIT_INVALID_INPUT)
+                elif type(value) in [CommentedMap, dict]:
+                    break_inheritance(value, parent_path + (key,))
+
+        def replace_config_section(merged_config, parent_path, specific_config):
+            """
+            Replace the merged section at ``parent_path`` with the specific section
+            that requested an inheritance break, dropping the control flag itself.
+            """
+            target_config = merged_config
+            for key in parent_path[:-1]:
+                target_config = target_config[key]
+
+            replacement_config = deepcopy(specific_config)
+            replacement_config.pop("inherit", None)
+            target_config[parent_path[-1]] = replacement_config
+
+        break_inheritance(more_specific_config)
+
+        return dict(merged_dict)
+
+    @staticmethod
+    def _get_case_insensitively(a_dict: dict, a_key: str):
+        for dict_key in a_dict.keys():
+            if dict_key.lower() == a_key.lower():
+                return a_dict[dict_key]
+        return {}
+
+    @staticmethod
+    def _is_skipped_case_insensitively(an_array: list, item: str) -> bool:
+        """
+        :return: if item is defined in the list to be skipped
+        """
+        item = item.lower()
+
+        for list_element in an_array:
+            list_element = list_element.lower()
+
+            if list_element == item:
+                return True
+
+            if (
+                list_element.endswith("/*")
+                and item.startswith(list_element[:-2])
+                and len(item) >= len(list_element[:-2])
+            ):
+                return True
+
+        return False
+
+    def _find_almost_duplicates(self):
+        # in GitLab groups and projects names are de facto case insensitive:
+        # you can change the case of both name and path BUT you cannot create
+        # 2 groups which names differ only with case and the same thing for
+        # projects. therefore we cannot allow such entries in the config,
+        # as they would be ambiguous.
+
+        for path in [
+            "projects_and_groups",
+            "skip_groups",
+            "skip_projects",
+        ]:
+            if self.get(path, 0):
+                almost_duplicates = self._find_almost_duplicates_in(path)
+                if almost_duplicates:
+                    critical(
+                        f"There are almost duplicates in the keys of {path} - they differ only in case.\n"
+                        f"They are: {', '.join(almost_duplicates)}\n"
+                        f"This is not allowed as we ignore the case for group and project names."
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+    def _find_almost_duplicates_in(self, configuration_path):
+        """
+        Checks given configuration key and reads its keys - if it is a dict - or elements - if it is a list.
+        Looks for items that are almost the same - they differ only in the case.
+        :param configuration_path: configuration path, f.e. "group_settings"
+        :return: list of items that have almost duplicates,
+                 or an empty list if none are found
+        """
+
+        dict_or_list = self.get(configuration_path)
+        if isinstance(dict_or_list, dict):
+            items = dict_or_list.keys()
+        else:
+            items = dict_or_list
+
+        items_with_lowercase_names = [x.lower() for x in items]
+
+        # casting these to sets will deduplicate the one with lowercase names
+        # lowering its cardinality if there were elements in it
+        # that before lowering differed only in case
+        if len(set(items)) != len(set(items_with_lowercase_names)):
+            # we have some almost duplicates, let's find them
+            almost_duplicates = []
+            for first_item in items:
+                occurrences = 0
+                for second_item in items_with_lowercase_names:
+                    if first_item.lower() == second_item.lower():
+                        occurrences += 1
+                        if occurrences == 2:
+                            almost_duplicates.append(first_item)
+                            break
+            return almost_duplicates
+
+        else:
+            return []
+
+
+class ConfigFileNotFoundException(Exception):
+    pass
+
+
+class ConfigInvalidException(Exception):
+    def __init__(self, underlying: Exception):
+        self.underlying = underlying
+
+
+class KeyNotFoundException(Exception):
+    __slots__ = "key"
+
+    def __init__(self, key: str):
+        if logging.getLogger().getEffectiveLevel() <= logging.DEBUG:
+            self.key = key
+        else:
+            critical(f"Unable to find the key: {key.replace('|', '.')}\n")
+            sys.exit(EXIT_INVALID_INPUT)

gitlabform/configuration/groups.py

@@ -0,0 +1,127 @@
+from abc import ABC
+
+import functools
+from logging import debug
+
+from gitlabform.configuration import ConfigurationCommon
+from gitlabform.util import to_str
+
+
+class ConfigurationGroups(ConfigurationCommon, ABC):
+    """
+    Gets the groups, skipped groups and their effective configuration.
+
+    ConfigurationCommon is an ancestor of this class because the effective group configuration
+    depends on the common configuration.
+    """
+
+    def get_groups(self) -> list:
+        """
+        :return: sorted list of groups that are EXPLICITLY defined in the config
+        """
+        groups = []
+        projects_and_groups = self.get("projects_and_groups")
+        for element in projects_and_groups.keys():
+            if element.endswith("/*"):
+                # cut off that "/*"
+                group_name = element[:-2]
+                groups.append(group_name)
+        return sorted(groups)
+
+    def is_group_skipped(self, group):
+        """
+        :return: if group is defined in the key with groups to skip,
+                 ignoring the case
+        """
+        return self._is_skipped_case_insensitively(self.get("skip_groups", []), group)
+
+    @functools.lru_cache()
+    def get_effective_config_for_group(self, group) -> dict:
+        """
+        :param group: "group_name"
+        :return: merged configuration for this group, from common, group. Merging is additive.
+        """
+        # Check if group must be skipped
+        if self.is_group_skipped(group):
+            debug(f"Group {group} is skipped, returning empty config")
+            return {}
+
+        common_config = self.get_common_config()
+        debug("*Effective* common config: %s", to_str(common_config))
+
+        if "/" in group:
+            group_config = self._get_effective_subgroup_config(group)
+        else:
+            group_config = self._get_group_config(group)
+        debug("*Effective* group/subgroup config: %s", to_str(group_config))
+
+        if not common_config and group_config:
+            self._validate_break_inheritance_flag(group_config, group)
+
+        effective_config_for_group = self._merge_configs(common_config, group_config)
+        debug(
+            "*Effective* config common+group/subgroup: %s",
+            to_str(effective_config_for_group),
+        )
+
+        return effective_config_for_group
+
+    def _get_effective_subgroup_config(self, subgroup):
+        #
+        # Goes through a subgroups hierarchy, from top to bottom
+        #
+        # "x/y/x" -> ["x", "x/y", "x/y/z"]
+        #
+        # ...and for each element after 1st generate effective config from previous effective one merged with current:
+        #
+        #              |     v       |
+        #              \---> a       |
+        #                    |       v
+        #                    \------>b = effective config to return
+        #
+        # ...where a = merged_config("x", "x/y") and b = merged_config(a, "x/y/z")
+        #
+
+        effective_config = {}
+        elements = subgroup.split("/")
+        last_element = None
+        for element in elements:
+            if not last_element:
+                effective_config = self._get_group_config(element)
+                debug("First level config for '%s': %s", element, to_str(effective_config))
+                last_element = element
+            else:
+                next_level_subgroup = last_element + "/" + element
+                next_level_subgroup_config = self._get_group_config(next_level_subgroup)
+                debug(
+                    "Config for '%s': %s",
+                    next_level_subgroup,
+                    to_str(next_level_subgroup_config),
+                )
+
+                if effective_config:
+                    self._validate_break_inheritance_flag(effective_config, subgroup)
+                elif not effective_config and next_level_subgroup_config:
+                    self._validate_break_inheritance_flag(next_level_subgroup_config, next_level_subgroup)
+
+                effective_config = self._merge_configs(
+                    effective_config,
+                    next_level_subgroup_config,
+                )
+                debug(
+                    "Merged previous level config for '%s' with config for '%s': %s",
+                    last_element,
+                    next_level_subgroup,
+                    to_str(effective_config),
+                )
+                last_element = last_element + "/" + element
+
+        return effective_config
+
+    def _get_group_config(self, group) -> dict:
+        """
+        :param group: group/subgroup
+        :return: configuration for this group/subgroup or empty dict if not defined,
+                 ignoring the case
+        """
+        return self._get_case_insensitively(self.get("projects_and_groups"), f"{group}/*")

gitlabform/configuration/projects.py

@@ -0,0 +1,73 @@
+from abc import ABC
+
+import functools
+from logging import debug
+
+from gitlabform.configuration import ConfigurationGroups
+from gitlabform.util import to_str
+
+
+class ConfigurationProjects(ConfigurationGroups, ABC):
+    """
+    Gets the projects, skipped projects and their effective configuration.
+
+    ConfigurationGroups is an ancestor of this class because the effective project configuration
+    depends on the groups (and common) configuration.
+    """
+
+    def get_projects(self) -> list:
+        """
+        :return: sorted list of projects names, that are EXPLICITLY defined in the config
+        """
+        projects = []
+        projects_and_groups = self.get("projects_and_groups")
+        for element in projects_and_groups.keys():
+            if element != "*" and not element.endswith("/*"):
+                projects.append(element)
+        return sorted(projects)
+
+    def is_project_skipped(self, project) -> bool:
+        """
+        :return: if project is defined in the key with projects to skip,
+                 ignoring the case
+        """
+        return self._is_skipped_case_insensitively(self.get("skip_projects", []), project)
+
+    @functools.lru_cache()
+    def get_effective_config_for_project(self, group_and_project) -> dict:
+        """
+        :param group_and_project: "project_group/project_name"
+        :return: merged configuration for this project, from common, group/subgroup and project level.
+                 If project belongs to a subgroup, like "x/y/z", then it gets config from both group "x" as well
+                 as subgroup "y".
+                 Merging is additive.
+        """
+
+        group, _ = group_and_project.rsplit("/", 1)
+
+        effective_config_for_group = self.get_effective_config_for_group(group)
+
+        project_config = self._get_project_config(group_and_project)
+        debug("Project config: %s", to_str(project_config))
+
+        if not effective_config_for_group and project_config:
+            self._validate_break_inheritance_flag(project_config, group_and_project)
+
+        effective_config_for_project = self._merge_configs(
+            effective_config_for_group,
+            project_config,
+        )
+        debug(
+            "*Effective* config common+group/subgroup+project: %s",
+            to_str(effective_config_for_project),
+        )
+
+        return effective_config_for_project
+
+    def _get_project_config(self, group_and_project) -> dict:
+        """
+        :param group_and_project: 'group/project'
+        :return: configuration for this project or empty dict if not defined,
+                 ignoring the case
+        """
+        return self._get_case_insensitively(self.get("projects_and_groups"), group_and_project)