gitlabform 6.0.0__tar.gz → 6.1.0__tar.gz

{gitlabform-6.0.0 → gitlabform-6.1.0}/.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,3 +18,14 @@ Output of `gitlabform -V`
 **GitLab version**
 
 F.e. `13.12.4-ee`
+
+**GitLabForm configuration**
+
+The relevant part of your gitlabform config (optionally anonymized).  
+If applicable, share both the before and after config.
+
+**Would you be open to contributing a fix?**
+
+- [ ] Yes
+- [ ] No
+- [ ] Maybe

{gitlabform-6.0.0 → gitlabform-6.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitlabform
-Version: 6.0.0
+Version: 6.1.0
 Summary: 🏗 Specialized configuration as a code tool for GitLab projects, groups and more using hierarchical configuration written in YAML
 Project-URL: Homepage, https://gitlabform.github.io/gitlabform/
 Project-URL: Repository, https://github.com/gitlabform/gitlabform.git
@@ -19,7 +19,7 @@ Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Version Control :: Git
 Requires-Python: >=3.12.0
-Requires-Dist: certifi==2026.5.20
+Requires-Dist: certifi==2026.6.17
 Requires-Dist: cli-ui==0.19.0
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: luddite==1.0.4

{gitlabform-6.0.0 → gitlabform-6.1.0}/dev/gitlab/gitlab.rb

@@ -1,4 +1,7 @@
 gitlab_rails['initial_root_password']='mK9JnG7jwYdFcBNoQ3W3'
+# Bind nginx on IPv4 and IPv6 loopback. Without IPv6, Sidekiq's mirror sync
+# fails intermittently with ECONNREFUSED when libcurl resolves localhost to ::1.
+nginx['listen_addresses'] = ['0.0.0.0', '[::]']
 registry['enable']=false
 prometheus_monitoring['enable']=false
 gitlab_rails['initial_license_file']='/etc/gitlab/Gitlab.gitlab-license'
@@ -38,4 +41,4 @@ gitlab_rails['omniauth_providers'] = [
            },
     label: 'SAML Login'
   }
-]
+]

{gitlabform-6.0.0 → gitlabform-6.1.0}/docs/changelog.md

@@ -6,6 +6,24 @@ For details on how to migrate between major versions, please refer to the [upgra
 
 ---
 
+## 6.1.0
+
+### Features
+
+* Add wildcard pattern support for project configuration [#1311](https://github.com/gitlabform/gitlabform/pull/1311). ([nikaro](https://github.com/nikaro)) 
+
+### Bug Fixes
+
+* group_saml_links: stop delete-and-recreate churn on unchanged links [#1331](https://github.com/gitlabform/gitlabform/pull/1331). ([rickbrouwer](https://github.com/rickbrouwer))
+* validate key in files config to avoid cryptic KeyError [#1317](https://github.com/gitlabform/gitlabform/pull/1317), resolves [#1312](https://github.com/gitlabform/gitlabform/issues/1312). ([rickbrouwer](https://github.com/rickbrouwer))
+* Normalize GitLab URL while parsing configuration [#1338](https://github.com/gitlabform/gitlabform/pull/1338). ([schurzi](https://github.com/schurzi))
+
+### Dependencies
+
+* Update various dependencies to newer versions.
+
+Thanks to all the contributors of this release!
+
 ## 6.0.0
 
 ### Breaking Changes

{gitlabform-6.0.0 → gitlabform-6.1.0}/gitlabform/configuration/core.py

@@ -1,3 +1,4 @@
+import fnmatch
 import sys
 from typing import Any
 
@@ -211,6 +212,26 @@ class ConfigurationCore(ABC):
 
         return dict(merged_dict)
 
+    @staticmethod
+    def _get_key_type(a_key: str) -> str:
+        """
+        Classify a config key used under 'projects_and_groups'.
+        """
+        if a_key == "*":
+            return "common"
+        if a_key.endswith("/*"):
+            return "group"
+        if "*" in a_key:
+            return "project_pattern"
+        return "project"
+
+    @staticmethod
+    def _match_pattern(pattern: str, item: str) -> bool:
+        """
+        Case-insensitive fnmatch between pattern and item.
+        """
+        return fnmatch.fnmatchcase(item.lower(), pattern.lower())
+
     @staticmethod
     def _get_case_insensitively(a_dict: dict, a_key: str):
         for dict_key in a_dict.keys():
@@ -231,6 +252,7 @@ class ConfigurationCore(ABC):
             if list_element == item:
                 return True
 
+            # Support for traditional group/* skip pattern
             if (
                 list_element.endswith("/*")
                 and item.startswith(list_element[:-2])
@@ -238,6 +260,10 @@ class ConfigurationCore(ABC):
             ):
                 return True
 
+            # Support for partial wildcard patterns like group/foo-*
+            if "*" in list_element and fnmatch.fnmatchcase(item, list_element):
+                return True
+
         return False
 
     def _find_almost_duplicates(self):

{gitlabform-6.0.0 → gitlabform-6.1.0}/gitlabform/configuration/projects.py

@@ -15,14 +15,14 @@ class ConfigurationProjects(ConfigurationGroups, ABC):
     depends on the groups (and common) configuration.
     """
 
-    def get_projects(self) -> list:
+    def get_projects(self, key_type: str = "project") -> list:
         """
         :return: sorted list of projects names, that are EXPLICITLY defined in the config
         """
         projects = []
         projects_and_groups = self.get("projects_and_groups")
         for element in projects_and_groups.keys():
-            if element != "*" and not element.endswith("/*"):
+            if self._get_key_type(element) == key_type:
                 projects.append(element)
         return sorted(projects)
 
@@ -68,6 +68,27 @@ class ConfigurationProjects(ConfigurationGroups, ABC):
         """
         :param group_and_project: 'group/project'
         :return: configuration for this project or empty dict if not defined,
-                 ignoring the case
+                 ignoring the case. Also checks for matching project patterns.
         """
-        return self._get_case_insensitively(self.get("projects_and_groups"), group_and_project)
+        projects_and_groups = self.get("projects_and_groups")
+
+        # 1. Exact match
+        exact = self._get_case_insensitively(projects_and_groups, group_and_project)
+        if exact:
+            return exact
+
+        # 2. Best matching pattern
+        matches = []
+        for key in projects_and_groups.keys():
+            if self._get_key_type(key) == "project_pattern":
+                if self._match_pattern(key, group_and_project):
+                    # store a 3-tuple for later sorting
+                    prefix = key.split("*", 1)[0]
+                    matches.append((len(prefix), -key.count("*"), key))
+        if matches:
+            # longest literal prefix wins; tie -> fewer wildcards; still tie -> first appearance
+            matches.sort(reverse=True)
+            best_key = matches[0][2]
+            return self._get_case_insensitively(projects_and_groups, best_key)
+
+        return {}

{gitlabform-6.0.0 → gitlabform-6.1.0}/gitlabform/gitlab/core.py

@@ -37,6 +37,9 @@ class GitLabCore:
         gitlab_config_from_file = self.configuration.get("gitlab", {})
         self.gitlab_config = {**default_gitlab_config, **gitlab_config_from_file}
 
+        if self.gitlab_config["url"]:
+            self.gitlab_config["url"] = self.gitlab_config["url"].rstrip("/")
+
         self.session = requests.Session()
 
         retries_status_forcelist = []

{gitlabform-6.0.0 → gitlabform-6.1.0}/gitlabform/lists/projects.py

@@ -96,6 +96,13 @@ class ProjectsProvider(GroupsProvider):
             # defined in the configuration, but we don't need to re-check for
             # being archived or scheduled for deletion projects that we already got from groups
 
+            for resolved_project in self._resolve_project_patterns(groups.get_effective()):
+                if (
+                    resolved_project not in projects.requested
+                    and resolved_project not in projects_from_configuration_not_from_groups
+                ):
+                    projects_from_configuration_not_from_groups.append(resolved_project)
+
             (
                 archived_projects_from_configuration_not_from_groups,
                 scheduled_for_deletion_projects_from_configuration_not_from_groups,
@@ -198,6 +205,42 @@ class ProjectsProvider(GroupsProvider):
 
         return skipped
 
+    def _resolve_project_patterns(self, effective_groups: list) -> list:
+        """
+        Resolve project patterns from config to concrete project paths.
+        Only resolves patterns whose parent group is not already in the effective groups list
+        to avoid redundant API calls.
+        """
+        pattern_projects = []
+        for pattern in self.configuration.get_projects("project_pattern"):
+            pattern_group = pattern.rsplit("/", 1)[0]
+            if pattern_group in effective_groups:
+                # projects from this group already fetched through normal means
+                continue
+
+            try:
+                maybe_group = self.gitlab.get_group_case_insensitive(pattern_group)
+                group_path = maybe_group["full_path"]
+            except NotFoundException:
+                debug(
+                    "Group '%s' for pattern '%s' not found in GitLab, skipping",
+                    pattern_group,
+                    pattern,
+                )
+                continue
+
+            try:
+                all_project_objects = self.gitlab.get_projects(group_path, include_archived=True, only_names=False)
+            except NotFoundException:
+                all_project_objects = []
+
+            for project_object in all_project_objects:
+                path = project_object["path_with_namespace"]
+                if self.configuration._match_pattern(pattern, path) and path not in pattern_projects:
+                    pattern_projects.append(path)
+
+        return pattern_projects
+
     def _get_project_transfer_source_from_config(self, project: str) -> Optional[str]:
         try:
             debug(

gitlabform-6.1.0/gitlabform/processors/group/group_saml_links_processor.py

@@ -0,0 +1,61 @@
+from logging import debug
+from typing import List
+
+from gitlabform.gitlab import GitLab
+from gitlab.v4.objects import Group, GroupSAMLGroupLink
+from gitlabform.processors.abstract_processor import AbstractProcessor
+
+
+class GroupSAMLLinksProcessor(AbstractProcessor):
+
+    def __init__(self, gitlab: GitLab):
+        super().__init__("group_saml_links", gitlab)
+
+    def _process_configuration(self, group_path_and_name: str, configuration: dict) -> None:
+        """Process the SAML links configuration for a group."""
+
+        configured_section = configuration.get("group_saml_links", {}) or {}
+        enforce_links = configuration.get("group_saml_links|enforce", False)
+
+        configured_links = {k: v for k, v in configured_section.items() if k != "enforce"}
+
+        group: Group = self.gl.get_group_by_path_cached(group_path_and_name)
+        existing_links: List[GroupSAMLGroupLink] = group.saml_group_links.list(get_all=True)
+        existing_by_name = {link.name: link for link in existing_links}
+
+        for _, link_configuration in configured_links.items():
+            saml_group_name = link_configuration.get("saml_group_name")
+            existing_link = existing_by_name.get(saml_group_name)
+
+            if existing_link is None:
+                group.saml_group_links.create(link_configuration)
+                continue
+
+            # GitLab API's request and response attributes differs when SAML link is created vs retrieved.
+            # On creation or retrieving a link, attribute is called `saml_group_name` but and returned response is 'name' instead.
+            # To compare existing link (retrieved from GitLab) with configured link, link configuration need to replace `saml_group_name` with `name`
+            expected = {**link_configuration, "name": saml_group_name}
+            expected.pop("saml_group_name", None)
+
+            if self._needs_update(existing_link.asdict(), expected):
+                # GitLab API has no update endpoint for SAML links; delete and recreate instead.
+                debug(f"Updating SAML link: {saml_group_name} with {link_configuration}")
+                existing_link.delete()
+                group.saml_group_links.create(link_configuration)
+
+        if enforce_links:
+            self._delete_extra_links(group, existing_links, configured_links)
+
+    def _delete_extra_links(
+        self,
+        group: Group,
+        existing: List[GroupSAMLGroupLink],
+        configured: dict,
+    ) -> None:
+        """Delete any SAML links that are not in the configuration."""
+        known_names = {link["saml_group_name"] for link in configured.values()}
+
+        for link in existing:
+            if link.name not in known_names:
+                debug(f"Deleting extra SAML link: {link.name}")
+                group.saml_group_links.delete(link.name)

{gitlabform-6.0.0 → gitlabform-6.1.0}/gitlabform/processors/project/files_processor.py

@@ -24,6 +24,19 @@ class FilesProcessor(AbstractProcessor):
         self.strict = strict
         self.branch_processor = BranchesProcessor(gitlab, strict)
 
+    def _can_proceed(self, project_or_group: str, configuration: dict):
+        for file in sorted(configuration["files"]):
+            file_config = configuration["files"][file]
+            if "branches" not in file_config:
+                critical(
+                    f"File '{file}' is missing the required 'branches' key."
+                    f" Specify which branches to apply the file to using one of:"
+                    f" 'all', 'protected', or a list of branch names (e.g. ['main'])."
+                )
+                sys.exit(EXIT_INVALID_INPUT)
+
+        return True
+
     def _process_configuration(self, project_and_group: str, configuration: dict):
         for file in sorted(configuration["files"]):
             project: Project = self.gl.get_project_by_path_cached(project_and_group)

{gitlabform-6.0.0 → gitlabform-6.1.0}/pyproject.toml

@@ -10,7 +10,7 @@ packages = ["gitlabform"]
 
 [project]
 name = "gitlabform"
-version = "6.0.0"
+version = "6.1.0"
 requires-python = ">=3.12.0"
 authors = [{ name = "Greg Dubicki and Contributors" }]
 description = "🏗 Specialized configuration as a code tool for GitLab projects, groups and more using hierarchical configuration written in YAML"
@@ -31,7 +31,7 @@ classifiers = [
 
 # Main runtime dependencies. Pinned strictly to ensure production stability.
 dependencies = [
-    "certifi==2026.5.20",
+    "certifi==2026.6.17",
     "cli-ui==0.19.0",
     "jinja2==3.1.6",
     "luddite==1.0.4",
@@ -53,14 +53,14 @@ dev = [
     { include-group = "lint" },
     { include-group = "docs" },
     { include-group = "release" },
-    "commitizen==4.16.3",
-    "prek==0.4.4",
+    "commitizen==4.16.4",
+    "prek==0.4.5",
 ]
 # Minimum set of dependencies required to execute the test suite.
 test = [
-    "coverage==7.14.1",
+    "coverage==7.14.3",
     "cryptography==49.0.0",
-    "pytest==9.1.0",
+    "pytest==9.1.1",
     "pytest-cov==7.1.0",
     "pytest-rerunfailures==16.3",
     "xkcdpass==1.30.0",
@@ -72,7 +72,7 @@ lint = [
     "codespell==2.4.2",
     "mypy==2.1.0",
     "mypy-extensions==1.1.0",
-    "ruff==0.15.17",
+    "ruff==0.15.20",
     "types-requests==2.33.0.20260518",
 ]
 # Documentation generation stack.
@@ -82,7 +82,7 @@ docs = [
     "mkdocs-material==9.7.6",
     "mkdocs-material-extensions==1.3.1",
     "pygments==2.20.0",
-    "pymdown-extensions==10.21.3",
+    "pymdown-extensions==11.0",
 ]
 # Tools required for building and verifying releases.
 release = [

{gitlabform-6.0.0 → gitlabform-6.1.0}/tbump.toml

@@ -2,7 +2,7 @@
 github_url = "https://github.com/gitlabform/gitlabform/"
 
 [version]
-current = "6.0.0"
+current = "6.1.0"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

{gitlabform-6.0.0 → gitlabform-6.1.0}/tests/acceptance/premium/test_group_saml_links.py

@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 
 from tests.acceptance import run_gitlabform
@@ -46,6 +48,36 @@ class TestGroupSamlLinks:
         assert saml_links[0].name == "devops_users"
         assert saml_links[0].access_level == AccessLevel.get_value("developer")
 
+    def test__no_churn_when_config_unchanged(self, gl, group_for_function, caplog):
+        assert len(group_for_function.saml_group_links.list()) == 0, "group_saml_links is not empty"
+
+        config = f"""
+        projects_and_groups:
+          {group_for_function.full_path}/*:
+             group_saml_links:
+               devops_users:
+                 saml_group_name: devops_users
+                 access_level: maintainer
+        """
+
+        # First run creates the link.
+        run_gitlabform(config, group_for_function)
+        assert len(gl.groups.get(group_for_function.id).saml_group_links.list(get_all=True)) == 1
+
+        # Second run with the same config must not trigger a delete + recreate.
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG):
+            run_gitlabform(config, group_for_function)
+
+        assert (
+            "Updating SAML link" not in caplog.text
+        ), f"Expected no SAML link updates on second run, got: {caplog.text}"
+
+        saml_links = gl.groups.get(group_for_function.id).saml_group_links.list(get_all=True)
+        assert len(saml_links) == 1
+        assert saml_links[0].name == "devops_users"
+        assert saml_links[0].access_level == AccessLevel.get_value("maintainer")
+
     def test__enforce_saml_links(self, gl, group_for_function):
 
         assert len(group_for_function.saml_group_links.list()) == 0, "group_saml_links is not empty"

{gitlabform-6.0.0 → gitlabform-6.1.0}/tests/acceptance/standard/test_remote_mirrors.py

@@ -3,9 +3,9 @@ import os
 import pytest
 import time
 from typing import cast, Tuple, Generator
-from gitlab.v4.objects import Project, ProjectRemoteMirror, ProjectBranch, Group
+from gitlab.v4.objects import Project, ProjectRemoteMirror, Group
 
-from tests.acceptance import run_gitlabform, get_random_name, create_project
+from tests.acceptance import run_gitlabform, get_random_name, create_project, gl
 
 
 @pytest.fixture(scope="class")
@@ -349,86 +349,101 @@ class TestRemoteMirrorsProcessor:
 
     def test_remote_mirror_http_password_auth_sync(
         self,
-        project: Project,
-        mirror_urls: Tuple[str, str, str],
-        mirror_target_projects: Tuple[Project, Project, Project],
+        project_for_function: Project,
+        other_group: Group,
+        root_username: str,
+        root_access_token: str,
     ) -> None:
         """
         Test remote mirror functionality for http and password based authentication.
         This also tests configuration of `force_push` config key.
 
-        This is validated by creating a new branch and a test file in the source project.
-        Then run gitlabform with `force_push` enabled for the target mirror.
-        Then check the target mirror repo if the new branch and the test file exists.
+        Uses a function-scoped source and a dedicated empty target (no initial commit, so no
+        default-branch auto-protection race) to keep this test self-contained and free of the
+        cross-test state that previously made it flaky.
         """
-        first_mirror_url_http_password_auth, _, _ = mirror_urls
-        first_mirror_repo, _, _ = mirror_target_projects
-
-        # In GitLab, default setting for branch protection is to deny force push.
-        # For first mirror, we need to allow force push to the main branch, so that
-        # we can validate the mirror functionality.
-        first_mirror_repo_main_branch = cast(ProjectBranch, first_mirror_repo.protectedbranches.get("main"))
-        first_mirror_repo_main_branch.allow_force_push = True
-        first_mirror_repo_main_branch.save()
-
-        # Create a test file in a new branch in the source project to validate mirror functionality
-        # We use a unique branch name to ensure we are testing a fresh sync
-        sync_branch_name = get_random_name("sync_test_branch")
-        project.branches.create({"branch": sync_branch_name, "ref": "main"})
+        # Empty target: no commits, no main branch yet, no protected-branch race. The mirror's
+        # first push creates main fresh from the source.
+        target = cast(
+            Project,
+            gl.projects.create(
+                {
+                    "name": get_random_name("mirror_sync_target"),
+                    "namespace_id": other_group.id,
+                    "default_branch": "main",
+                    "initialize_with_readme": False,
+                }
+            ),
+        )
+        try:
+            gitlab_url = os.getenv("GITLAB_URL", "http://localhost").rstrip("/")
+            mirror_url = (
+                gitlab_url.replace("http://", f"http://{root_username}:{root_access_token}@")
+                + f"/{target.path_with_namespace}.git"
+            )
 
-        test_file_content = "This is a test file for remote mirror validation"
-        test_file_path = f"mirror_test_{sync_branch_name}.txt"
+            sync_branch_name = get_random_name("sync_test_branch")
+            project_for_function.branches.create({"branch": sync_branch_name, "ref": "main"})
 
-        project.files.create(
-            {
-                "branch": sync_branch_name,
-                "file_path": test_file_path,
-                "content": test_file_content,
-                "commit_message": "Add test file for mirror validation",
-            }
-        )
+            test_file_content = "This is a test file for remote mirror validation"
+            test_file_path = f"mirror_test_{sync_branch_name}.txt"
 
-        # Validate that the test file exists in the source project
-        source_file = project.files.get(ref=sync_branch_name, file_path=test_file_path)
-        assert source_file is not None
-        assert source_file.decode().decode("utf-8") == test_file_content
+            project_for_function.files.create(
+                {
+                    "branch": sync_branch_name,
+                    "file_path": test_file_path,
+                    "content": test_file_content,
+                    "commit_message": "Add test file for mirror validation",
+                }
+            )
 
-        # Run GitLabForm with force_push enabled.
-        # This should trigger the self._sync_remote_mirror(project, mirror_in_gitlab) logic.
-        gitlabform_config = f"""
-            projects_and_groups:
-              {project.path_with_namespace}:
-                remote_mirrors:
-                  {first_mirror_url_http_password_auth}:
-                    enabled: true
-                    auth_method: password
-                    force_push: true
-            """
+            gitlabform_config = f"""
+                projects_and_groups:
+                  {project_for_function.path_with_namespace}:
+                    remote_mirrors:
+                      {mirror_url}:
+                        enabled: true
+                        auth_method: password
+                        force_push: true
+                """
 
-        run_gitlabform(gitlabform_config, project.path_with_namespace)
+            run_gitlabform(gitlabform_config, project_for_function.path_with_namespace)
 
-        # Validate mirror functionality: wait for sync and verify test file appears in target.
-        # Even with force_push: true, GitLab's Sidekiq worker may take a few seconds to
-        # complete the physical git push to the remote.
-        found = False
-        max_retries = 10
-        retry_interval = 5
-
-        for i in range(max_retries):
-            try:
-                # Attempt to retrieve the file from the target repository
-                target_file = first_mirror_repo.files.get(ref=sync_branch_name, file_path=test_file_path)
-                if target_file.decode().decode("utf-8") == test_file_content:
-                    found = True
-                    break
-            except Exception:
-                # If 404/Exception, wait and try again
-                time.sleep(retry_interval)
-
-        assert found, (
-            f"Mirror sync failed: File '{test_file_path}' not found in target project "
-            f"'{first_mirror_repo.path_with_namespace}' after {max_retries * retry_interval} seconds."
-        )
+            max_wait_seconds = 60
+            poll_interval = 3
+            deadline = time.time() + max_wait_seconds
+            last_status: str | None = None
+            last_error: str | None = None
+            synced = False
+
+            while time.time() < deadline:
+                mirror = self._get_mirror_from_url(project_for_function, mirror_url)
+                assert mirror is not None, "Mirror disappeared from source project during sync"
+                last_status = getattr(mirror, "update_status", None)
+                last_error = getattr(mirror, "last_error", None)
+
+                if last_status == "failed":
+                    pytest.fail(f"Mirror sync reported failed by GitLab. last_error={last_error!r}")
+
+                if last_status == "finished":
+                    try:
+                        target_file = target.files.get(ref=sync_branch_name, file_path=test_file_path)
+                        if target_file.decode().decode("utf-8") == test_file_content:
+                            synced = True
+                            break
+                    except Exception:
+                        pass
+
+                time.sleep(poll_interval)
+
+            assert synced, (
+                f"Mirror sync did not complete within {max_wait_seconds}s: "
+                f"file '{test_file_path}' not found in target project "
+                f"'{target.path_with_namespace}' "
+                f"(last update_status={last_status!r}, last_error={last_error!r})."
+            )
+        finally:
+            target.delete()
 
     def test_remote_mirrors_sync_error(
         self,