gitlabform 0.0.540a0__py3-none-any.whl

gitlabform/lists/__init__.py

@@ -0,0 +1,62 @@
+import enum
+
+import copy
+from collections import defaultdict
+
+from typing import DefaultDict
+
+from abc import ABC
+
+from enum import Enum
+
+
+@enum.unique
+class OmissionReason(Enum):
+    ARCHIVED = "archived"
+    SCHEDULED_FOR_DELETION = "scheduled for deletion"
+    EMPTY = "empty effective config"
+    SKIPPED = "skipped"
+
+
+class Entities(ABC):
+    """
+    Entities here are lists of GitLab groups or projects on which GitLabForm will operate.
+    The code here assumes that there is a set of requested entities, but some of them may be omitted
+    for various reasons, so we want to be able to get only the effective set of them.
+    """
+
+    def __init__(self, name: str):
+        self.requested: set = set()
+        self.omitted: DefaultDict[OmissionReason, set] = defaultdict(set)
+        self.name = name
+
+    def add_requested(self, more_requested: list) -> None:
+        self.requested = self.requested | set(more_requested)
+
+    def add_omitted(self, reason: OmissionReason, more_omitted: list) -> None:
+        self.omitted[reason] = self.omitted[reason] | set(more_omitted)
+
+    def get_omitted(self, reason: OmissionReason) -> list:
+        return sorted(self.omitted[reason])
+
+    def any_omitted(self) -> bool:
+        for reason in self.omitted:
+            if len(self.omitted[reason]) > 0:
+                return True
+        return False
+
+    def get_effective(self) -> list:
+        effective_set = copy.deepcopy(self.requested)
+        for reason in self.omitted:
+            effective_set -= self.omitted[reason]
+        return sorted(effective_set)
+
+
+class Groups(Entities):
+    def __init__(self):
+        super().__init__("groups")
+
+
+class Projects(Entities):
+    def __init__(self):
+        super().__init__("projects")

gitlabform/lists/filter.py

@@ -0,0 +1,99 @@
+import sys
+from abc import ABC, abstractmethod
+
+from logging import critical
+
+from gitlabform.constants import EXIT_INVALID_INPUT
+from gitlabform.lists import OmissionReason, Groups, Projects
+
+# Groups and projects filters jobs is to omit some groups and projects that GitLabForm is requested
+# to process for a speed-up.
+#
+# An example reason for omitting groups and projects is when they have an empty effective config.
+
+
+class GroupsAndProjectsFilters:
+    def __init__(self, configuration, group_processors, project_processors):
+        self.configuration = configuration
+
+        self.omit_empty_configs = OmitEmptyConfigs(configuration, group_processors, project_processors)
+        # add next filters here
+
+    def filter(self, groups: Groups, projects: Projects):
+        self.omit_empty_configs.filter(groups, projects)
+        # add next filters here
+
+
+class GroupsAndProjectsFilter(ABC):
+    @abstractmethod
+    def filter(self, groups: Groups, projects: Projects):
+        pass
+
+
+class OmitEmptyConfigs(GroupsAndProjectsFilter):
+    """
+    One of the reasons groups or project can be omitted from processing is when they have an empty effective config.
+
+    For example with a config like:
+
+    projects_and_groups:
+      *:
+        group_variables:
+          foobar:
+            key: "a key"
+            value: "the value"
+
+    ...and a request query "foo/bar" that points to a project "bar" in a group "foo", the effective config
+    is empty, as "group_variables" is a group-level configuration and "foo/bar" is a project.
+    """
+
+    def __init__(self, configuration, group_processors, project_processors):
+        self.configuration = configuration
+        self.group_processors = group_processors
+        self.project_processors = project_processors
+
+        if not self.configuration.get("projects_and_groups", {}):
+            critical("Configuration has to contain non-empty 'projects_and_groups' key.")
+            sys.exit(EXIT_INVALID_INPUT)
+
+    def filter(self, groups: Groups, projects: Projects) -> None:
+        """
+        :param groups: list of groups (and possibly subgroups)
+        :param projects: list of projects
+        """
+
+        groups_with_empty_configs = []
+        for group in groups.get_effective():
+            if self._group_has_empty_effective_config(group):
+                groups_with_empty_configs.append(group)
+        groups.add_omitted(OmissionReason.EMPTY, groups_with_empty_configs)
+
+        projects_with_empty_configs = []
+        for project in projects.get_effective():
+            if self._project_has_empty_effective_config(project):
+                projects_with_empty_configs.append(project)
+        projects.add_omitted(OmissionReason.EMPTY, projects_with_empty_configs)
+
+    def _group_has_empty_effective_config(self, group: str) -> bool:
+        """
+        :param group: group/subgroup
+        :return: if given group/subgroup has no config that can be processed
+                 by any group-level processors
+        """
+        config_for_group = self.configuration.get_effective_config_for_group(group)
+        for configuration_name in config_for_group.keys():
+            if configuration_name in self.group_processors.get_configuration_names():
+                return False
+        return True
+
+    def _project_has_empty_effective_config(self, project: str) -> bool:
+        """
+        :param project: 'group/project'
+        :return: if given project has no config that can be processed
+                 by any project-level processors
+        """
+        config_for_project = self.configuration.get_effective_config_for_project(project)
+        for configuration_name in config_for_project.keys():
+            if configuration_name in self.project_processors.get_configuration_names():
+                return False
+        return True

gitlabform/lists/groups.py

@@ -0,0 +1,87 @@
+import sys
+from logging import critical
+from gitlabform.constants import EXIT_INVALID_INPUT
+from gitlabform.gitlab.core import NotFoundException
+from gitlabform.lists import OmissionReason, Groups
+
+
+class GroupsProvider:
+    """
+    For a query like "project/group", "group", "group/subgroup", ALL or ALL_DEFINED this
+    class gets the effective lists of groups, taking into account skipped groups
+    and the fact that the group and project names case are somewhat case-sensitive.
+    """
+
+    def __init__(self, gitlab, configuration, recurse_subgroups):
+        self.gitlab = gitlab
+        self.configuration = configuration
+        self.recurse_subgroups = recurse_subgroups
+
+    def get_groups(self, target: str) -> Groups:
+        """
+        :param target: "project/group", "group", "group/subgroup", ALL or ALL_DEFINED
+        :return: Groups
+        """
+
+        if target not in ["ALL", "ALL_DEFINED"]:
+            groups = self._get_single_group(target, self.recurse_subgroups)
+        else:
+            groups = self._get_groups(target)
+
+        return groups
+
+    def _get_single_group(self, target: str, recurse_subgroups: bool) -> Groups:
+        groups = Groups()
+
+        # it may be a subgroup or a group...
+        try:
+            maybe_group = self.gitlab.get_group_case_insensitive(target)
+            path = maybe_group["full_path"]
+            groups.add_requested([path])
+
+            if recurse_subgroups:
+                descendants = self.gitlab.get_group_descendants(path)
+                groups.add_requested([group["full_path"] for group in descendants])
+
+                groups.add_omitted(
+                    OmissionReason.SKIPPED,
+                    self._get_skipped_groups(groups.get_effective()),
+                )
+
+        except NotFoundException:
+            # ...or a single project, which we ignore here
+            pass
+
+        return groups
+
+    def _get_groups(self, target: str) -> Groups:
+        groups = Groups()
+
+        if target == "ALL":
+            groups.add_requested(self.gitlab.get_groups())
+        else:  # ALL_DEFINED
+            groups.add_requested(self.configuration.get_groups())
+
+        groups.add_omitted(OmissionReason.SKIPPED, self._get_skipped_groups(groups.get_effective()))
+
+        if target == "ALL_DEFINED":
+            # check if all the groups from the config actually exist
+            self._verify_if_groups_exist(groups.get_effective())
+
+        return groups
+
+    def _verify_if_groups_exist(self, groups: list):
+        for group in groups:
+            try:
+                self.gitlab.get_group_case_insensitive(group)
+            except NotFoundException:
+                critical(f"Configuration contains group {group} but it cannot be found in GitLab!")
+                sys.exit(EXIT_INVALID_INPUT)
+
+    def _get_skipped_groups(self, groups: list) -> list:
+        skipped = []
+        for group in groups:
+            if self.configuration.is_group_skipped(group):
+                skipped.append(group)
+
+        return skipped

gitlabform/lists/projects.py

@@ -0,0 +1,239 @@
+import sys
+from typing import Optional, Tuple
+from logging import debug
+from logging import critical
+
+from gitlabform.constants import EXIT_INVALID_INPUT
+from gitlabform.lists import OmissionReason, Groups, Projects
+from gitlabform.lists.groups import GroupsProvider
+
+from gitlabform.gitlab.core import NotFoundException
+
+
+class ProjectsProvider(GroupsProvider):
+    """
+    For a query like "project/group", "group", "group/subgroup", ALL or ALL_DEFINED this
+    class gets the effective lists of projects, taking into account skipped projects
+    and the fact that the group and project names case are somewhat case-sensitive.
+
+    Because the projects depend on groups requested, this class inherits GroupsProvider.
+    """
+
+    def __init__(
+        self,
+        gitlab,
+        configuration,
+        include_archived_projects,
+        include_projects_scheduled_for_deletion,
+        recurse_subgroups,
+    ):
+        super().__init__(gitlab, configuration, recurse_subgroups)
+        self.include_archived_projects = include_archived_projects
+        self.include_projects_scheduled_for_deletion = include_projects_scheduled_for_deletion
+
+    def get_projects(self, target: str) -> Projects:
+        """
+        :param target: "project/group", "group", "group/subgroup", ALL or ALL_DEFINED
+        :return: Projects
+        """
+
+        groups = self.get_groups(target)
+
+        if target not in ["ALL", "ALL_DEFINED"]:
+            if len(groups.get_effective()) == 1:
+                projects = self._get_projects(target, groups)
+            else:
+                projects = self._get_single_project(target)
+        else:
+            projects = self._get_projects(target, groups)
+
+        return projects
+
+    def _get_single_project(self, target: str) -> Projects:
+        projects = Projects()
+
+        # it may be a single project
+        try:
+            maybe_project = self.gitlab.get_project_case_insensitive(target)
+            projects.add_requested([maybe_project["path_with_namespace"]])
+
+        except NotFoundException:
+            debug("Could not find '%s'", target)
+            if project_transfer_source := self._get_project_transfer_source_from_config(target):
+                if self._find_project_transfer_source_in_gitlab(project_transfer_source):
+                    projects.add_requested([target])
+                else:
+                    critical(
+                        f"""Configuration contains project {target} to be transferred from {project_transfer_source}
+                            but the source project cannot be found in GitLab!"""
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+        return projects
+
+    def _get_projects(self, target: str, groups: Groups) -> Projects:
+        projects = Projects()
+
+        # the source of projects are the *effective* requested groups
+        (
+            projects_from_groups,
+            archived_projects_from_groups,
+            scheduled_for_deletion_projects_from_groups,
+        ) = self._get_all_and_omitted_projects_from_groups(groups.get_effective())
+        projects.add_requested(projects_from_groups)
+        projects.add_omitted(OmissionReason.ARCHIVED, archived_projects_from_groups)
+        projects.add_omitted(OmissionReason.SCHEDULED_FOR_DELETION, scheduled_for_deletion_projects_from_groups)
+
+        projects_from_configuration = self.configuration.get_projects()
+
+        # TODO: this check should be case-insensitive
+        projects_from_configuration_not_from_groups = [
+            project for project in projects_from_configuration if project not in projects.requested
+        ]
+
+        if target == "ALL_DEFINED":
+            # in this case we also need to get the list of projects explicitly
+            # defined in the configuration, but we don't need to re-check for
+            # being archived or scheduled for deletion projects that we already got from groups
+
+            (
+                archived_projects_from_configuration_not_from_groups,
+                scheduled_for_deletion_projects_from_configuration_not_from_groups,
+            ) = self._verify_if_projects_exist_and_get_omitted_projects(projects_from_configuration_not_from_groups)
+
+            projects.add_requested(projects_from_configuration_not_from_groups)
+            projects.add_omitted(
+                OmissionReason.ARCHIVED,
+                archived_projects_from_configuration_not_from_groups,
+            )
+            projects.add_omitted(
+                OmissionReason.SCHEDULED_FOR_DELETION,
+                scheduled_for_deletion_projects_from_configuration_not_from_groups,
+            )
+        else:
+            # in all other cases, we also need to look for projects in the config
+            # that are being transferred to a different namespace
+            for project in projects_from_configuration_not_from_groups:
+                # if the target is a group, and the project is not in the group
+                if len(groups.get_effective()) == 1 and target != "ALL" and target != project.rsplit("/", 1)[0]:
+                    debug(
+                        "Ignore project '%s', since it's not in target group '%s",
+                        project,
+                        target,
+                    )
+                else:
+                    if (maybe_projects := self._get_single_project(project)) and len(
+                        maybe_projects.get_effective()
+                    ) != 0:
+                        projects.add_requested([project])
+
+        # TODO: consider checking for skipped earlier to avoid making requests for projects that will be skipped anyway
+        projects.add_omitted(OmissionReason.SKIPPED, self._get_skipped_projects(projects.get_effective()))
+
+        return projects
+
+    def _verify_if_projects_exist_and_get_omitted_projects(self, projects: list) -> Tuple[list, list]:
+        archived = []
+        scheduled_for_deletion = []
+        for project in projects:
+            try:
+                project_object = self.gitlab.get_project_case_insensitive(project)
+                if not self.include_archived_projects and project_object["archived"]:
+                    archived.append(project_object["path_with_namespace"])
+                if not self.include_projects_scheduled_for_deletion and project_object.get("marked_for_deletion_on"):
+                    scheduled_for_deletion.append(project_object["path_with_namespace"])
+            except NotFoundException:
+                debug("Could not find '%s'", project)
+                if project_transfer_source := self._get_project_transfer_source_from_config(project):
+                    source_project = self._find_project_transfer_source_in_gitlab(project_transfer_source)
+                    if source_project:
+                        if not self.include_archived_projects and source_project["archived"]:
+                            archived.append(project)
+                        if not self.include_projects_scheduled_for_deletion and source_project.get(
+                            "marked_for_deletion_on"
+                        ):
+                            scheduled_for_deletion.append(project)
+                    else:
+                        critical(
+                            f"""Configuration contains project {project} to be transferred from {project_transfer_source}
+                                but the source project cannot be found in GitLab!"""
+                        )
+                        sys.exit(EXIT_INVALID_INPUT)
+                else:
+                    critical(
+                        f"Configuration contains project {project} but it cannot be found in GitLab and it's not a project that is configured to be transferred from elsewhere."
+                    )
+                    sys.exit(EXIT_INVALID_INPUT)
+
+        return archived, scheduled_for_deletion
+
+    def _get_all_and_omitted_projects_from_groups(self, groups: list) -> Tuple[list, list, list]:
+        all = []
+        archived = []
+        scheduled_for_deletion = []
+        for group in groups:
+            if self.include_archived_projects and self.include_projects_scheduled_for_deletion:
+                all += self.gitlab.get_projects(group, include_archived=True)
+            else:
+                project_objects = self.gitlab.get_projects(group, include_archived=True, only_names=False)
+                for project_object in project_objects:
+                    project = project_object["path_with_namespace"]
+                    all.append(project)
+                    if not self.include_archived_projects and project_object["archived"]:
+                        archived.append(project)
+                    if not self.include_projects_scheduled_for_deletion and project_object.get(
+                        "marked_for_deletion_on"
+                    ):
+                        scheduled_for_deletion.append(project)
+
+        # deduplicate as we may have a group X and its subgroup X/Y in the groups list so the effective projects
+        # may occur more than once
+        return list(set(all)), list(set(archived)), list(set(scheduled_for_deletion))
+
+    def _get_skipped_projects(self, projects: list) -> list:
+        skipped = []
+        for project in projects:
+            if self.configuration.is_project_skipped(project):
+                skipped.append(project)
+
+        return skipped
+
+    def _get_project_transfer_source_from_config(self, project: str) -> Optional[str]:
+        try:
+            debug(
+                "Checking if project '%s' needs to be transferred from elsewhere",
+                project,
+            )
+            project_transfer_source = self.configuration.config["projects_and_groups"][project]["project"][
+                "transfer_from"
+            ]
+            debug(
+                "Project '%s' needs to be transferred from '%s'",
+                project,
+                project_transfer_source,
+            )
+        except KeyError:
+            debug("'%s' is not a project needing to be transferred", project)
+            return None
+
+        return project_transfer_source
+
+    def _find_project_transfer_source_in_gitlab(self, project_transfer_source: str) -> Optional[dict]:
+        try:
+            debug(
+                "Checking if project transfer source ('%s') exists in GitLab",
+                project_transfer_source,
+            )
+            maybe_project = self.gitlab.get_project_case_insensitive(project_transfer_source)
+            debug(
+                "Project transfer source ('%s') exists",
+                project_transfer_source,
+            )
+        except NotFoundException:
+            debug(
+                "Project transfer source ('%s') does not exist",
+                project_transfer_source,
+            )
+            return None
+
+        return maybe_project

gitlabform/output.py

@@ -0,0 +1,46 @@
+import sys
+from logging import debug, critical, info
+
+import ez_yaml
+
+from gitlabform.constants import EXIT_INVALID_INPUT, EXIT_PROCESSING_ERROR
+
+
+class EffectiveConfigurationFile:
+    """
+    For GitLabForm upgrades and configuration refactoring we want to be able to compare the effective configurations
+    before and after the code/app change. This class provides a feature to write the effective configuration into a YAML
+    file.
+    """
+
+    def __init__(self, output_file):
+        if output_file:
+            try:
+                self.output_file = open(output_file, "w")
+                debug(f"Opened file {self.output_file} to write the effective configs to.")
+            except Exception as e:
+                critical(f"Error when trying to open {self.output_file} to write the effective configs to: {e}")
+                sys.exit(EXIT_INVALID_INPUT)
+        else:
+            self.output_file = None
+
+        self.config = {}
+
+    def add_placeholder(self, project_or_group: str):
+        if self.output_file:
+            self.config[project_or_group] = {}
+
+    def add_configuration(self, project_or_group: str, configuration_name: str, configuration: dict):
+        if self.output_file:
+            info(f"Adding effective configuration for {configuration_name}.")
+            self.config[project_or_group][configuration_name] = configuration
+
+    def write_to_file(self):
+        if self.output_file:
+            try:
+                yaml_configuration = ez_yaml.to_string(self.config)
+                self.output_file.write(yaml_configuration)
+                self.output_file.close()
+            except Exception as e:
+                critical(f"Error when trying to write or close {self.output_file}: {e}")
+                sys.exit(EXIT_PROCESSING_ERROR)

gitlabform/processors/__init__.py

@@ -0,0 +1,43 @@
+from abc import ABC
+
+from logging import info
+
+from typing import List
+
+from gitlabform.configuration import Configuration
+from gitlabform.gitlab import GitLab
+from gitlabform.output import EffectiveConfigurationFile
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class AbstractProcessors(ABC):
+    def __init__(self, gitlab: GitLab, config: Configuration, strict: bool):
+        self.processors: List[AbstractProcessor] = []
+
+    def get_configuration_names(self):
+        return [processor.configuration_name for processor in self.processors]
+
+    def process_entity(
+        self,
+        entity_reference: str,
+        configuration: dict,
+        dry_run: bool,
+        diff_only_changed: bool,
+        effective_configuration: EffectiveConfigurationFile,
+        only_sections: List[str] | str,
+        exclude_sections: List[str],
+    ):
+        for processor in self.processors:
+            if processor.configuration_name not in exclude_sections:
+                if only_sections == "all" or processor.configuration_name in only_sections:
+                    processor.process(
+                        entity_reference,
+                        configuration,
+                        dry_run,
+                        diff_only_changed,
+                        effective_configuration,
+                    )
+                else:
+                    info(f"Skipping section '{processor.configuration_name}' - not in --only-sections list.")
+            else:
+                info(f"Excluding section '{processor.configuration_name}'.")