qontract-reconcile 0.10.2.dev174__tar.gz → 0.10.2.dev176__tar.gz

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev174
+Version: 0.10.2.dev176
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/aus/base.py

@@ -970,56 +970,53 @@ def verify_schedule_should_skip(
     return next_schedule.strftime("%Y-%m-%dT%H:%M:%SZ")
 
 
-def verify_lock_should_skip(
-    desired: ClusterUpgradeSpec, locked: dict[str, str]
-) -> bool:
-    mutexes = desired.effective_mutexes
-    if any(lock in locked for lock in mutexes):
-        locking = {lock: locked[lock] for lock in mutexes if lock in locked}
-        logging.debug(
-            f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: locked out by {locking}"
-        )
-        return True
-    return False
-
-
 def verify_max_upgrades_should_skip(
     desired: ClusterUpgradeSpec,
-    sector_upgrades: dict[str, set[str]],
+    locked: dict[str, str],
+    sector_mutex_upgrades: dict[tuple[str, str], set[str]],
     sector: Sector | None,
 ) -> bool:
-    if sector is None:
-        return False
+    mutexes = desired.effective_mutexes
 
-    current_upgrades = sector_upgrades[sector.name]
-    # Allow at least one upgrade
-    if len(current_upgrades) == 0:
+    # if sector.max_parallel_upgrades is not set, we allow 1 upgrade per mutex, across the whole org
+    if sector is None or sector.max_parallel_upgrades is None:
+        if any(lock in locked for lock in mutexes):
+            locking = {lock: locked[lock] for lock in mutexes if lock in locked}
+            logging.debug(
+                f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: locked out by {locking}"
+            )
+            return True
         return False
 
-    # if sector.max_parallel_upgrades is not set, we allow all upgrades
-    if sector.max_parallel_upgrades is None:
-        return False
+    current_upgrades_count_per_mutex = {
+        mutex: len(sector_mutex_upgrades[sector.name, mutex]) for mutex in mutexes
+    }
 
-    sector_cluster_count = len(sector.specs)
+    current_upgrades_total_count = sum(current_upgrades_count_per_mutex.values())
+    if current_upgrades_total_count == 0:
+        return False
 
-    if sector.max_parallel_upgrades.endswith("%"):
-        max_parallel_upgrades_percent = int(sector.max_parallel_upgrades[:-1])
-        max_parallel_upgrades = round(
-            sector_cluster_count * max_parallel_upgrades_percent / 100
-        )
-    else:
-        max_parallel_upgrades = int(sector.max_parallel_upgrades)
+    for mutex in mutexes:
+        cluster_count = len([s for s in sector.specs if mutex in s.effective_mutexes])
+        if sector.max_parallel_upgrades.endswith("%"):
+            max_parallel_upgrades_percent = int(sector.max_parallel_upgrades[:-1])
+            max_parallel_upgrades = round(
+                cluster_count * max_parallel_upgrades_percent / 100
+            )
+        else:
+            max_parallel_upgrades = int(sector.max_parallel_upgrades)
 
-    # we allow at least one upgrade
-    if max_parallel_upgrades == 0:
-        max_parallel_upgrades = 1
+        # we allow at least one upgrade
+        if max_parallel_upgrades == 0:
+            max_parallel_upgrades = 1
 
-    if len(current_upgrades) >= max_parallel_upgrades:
-        logging.debug(
-            f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: "
-            f"sector '{sector.name}' has reached max parallel upgrades {sector.max_parallel_upgrades}"
-        )
-        return True
+        if current_upgrades_count_per_mutex.get(mutex, 0) >= max_parallel_upgrades:
+            logging.debug(
+                f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: "
+                f"sector '{sector.name}' has reached max parallel upgrades {sector.max_parallel_upgrades} "
+                f"for mutex '{mutex}'"
+            )
+            return True
 
     return False
 
@@ -1085,15 +1082,15 @@ def calculate_diff(
     """
 
     locked: dict[str, str] = {}
-    sector_upgrades: dict[str, set[str]] = defaultdict(set)
+    sector_mutex_upgrades: dict[tuple[str, str], set[str]] = defaultdict(set)
 
     def set_upgrading(
-        cluster_id: str, mutexes: set[str] | None, sector_name: str | None
+        cluster_id: str, mutexes: set[str], sector_name: str | None
     ) -> None:
-        for mutex in mutexes or set():
+        for mutex in mutexes:
             locked[mutex] = cluster_id
-        if sector_name:
-            sector_upgrades[sector_name].add(cluster_id)
+            if sector_name:
+                sector_mutex_upgrades[sector_name, mutex].add(cluster_id)
 
     diffs: list[UpgradePolicyHandler] = []
 
@@ -1113,7 +1110,9 @@ def calculate_diff(
         # Upgrading node pools, only required for Hypershift clusters
         # do this in the same loop, to skip cluster on node pool upgrade
         if spec.cluster.is_rosa_hypershift():
-            if verify_lock_should_skip(spec, locked):
+            if verify_max_upgrades_should_skip(
+                spec, locked, sector_mutex_upgrades, sector
+            ):
                 continue
 
             node_pool_update = _calculate_node_pool_diffs(spec, now)
@@ -1135,10 +1134,7 @@ def calculate_diff(
         if not next_schedule:
             continue
 
-        if verify_lock_should_skip(spec, locked):
-            continue
-
-        if verify_max_upgrades_should_skip(spec, sector_upgrades, sector):
+        if verify_max_upgrades_should_skip(spec, locked, sector_mutex_upgrades, sector):
             continue
 
         version = upgradeable_version(spec, version_data, sector)

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/gitlab_members.py

@@ -68,9 +68,7 @@ def get_current_state(
     """Get current gitlab group members for all managed groups."""
     return {
         g: CurrentStateSpec(
-            members={
-                u.username: u for u in gl.get_group_members(gitlab_groups_map.get(g))
-            },
+            members={u.username: u for u in gl.get_group_members(gitlab_groups_map[g])},
         )
         for g in instance.managed_groups
     }

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/gitlab_permissions.py

@@ -1,12 +1,12 @@
 import itertools
 import logging
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, cast
 
 from gitlab.exceptions import GitlabGetError
 from gitlab.v4.objects import (
-    GroupProject,
     Project,
+    SharedProject,
 )
 from sretoolbox.utils import threaded
 
@@ -51,9 +51,9 @@ class GroupPermissionHandler:
             for project_repo_url in filtered_project_repos
         }
         # get all projects shared with group
-        shared_projects = self.gl.get_items(self.group.shared_projects.list)
+        shared_projects = self.group.shared_projects.list(iterator=True)
         current_state = {
-            project.web_url: self.extract_group_spec(project)
+            project.web_url: self.extract_group_spec(cast(SharedProject, project))
             for project in shared_projects
         }
         self.reconcile(desired_state, current_state)
@@ -61,13 +61,14 @@ class GroupPermissionHandler:
     def filter_group_owned_projects(self, repos: list[str]) -> set[str]:
         # get only the projects that are owned by  group and its sub groups
         query = {"with_shared": False, "include_subgroups": True}
-        group_owned_projects = self.gl.get_items(
-            self.group.projects.list, query_parameters=query
+        group_owned_projects = self.group.projects.list(
+            query_parameters=query,
+            iterator=True,
         )
         group_owned_repo_list = {project.web_url for project in group_owned_projects}
         return set(repos) - group_owned_repo_list
 
-    def extract_group_spec(self, project: GroupProject) -> GroupSpec:
+    def extract_group_spec(self, project: SharedProject) -> GroupSpec:
         return next(
             GroupSpec(
                 group_name=self.group.name,

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/utils/github_api.py

@@ -54,9 +54,14 @@ class GithubRepositoryApi:
         Align with GitLabApi
         """
 
-    def get_repository_tree(self, ref: str = "master") -> list[dict[str, str]]:
+    def get_repository_tree(
+        self,
+        *,
+        ref: str = "master",
+        recursive: bool = False,
+    ) -> list[dict[str, str]]:
         tree_items = []
-        for item in self._repo.get_git_tree(sha=ref, recursive=True).tree:
+        for item in self._repo.get_git_tree(sha=ref, recursive=recursive).tree:
             tree_item = {"path": item.path, "name": Path(item.path).name}
             tree_items.append(tree_item)
         return tree_items

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/utils/gitlab_api.py

@@ -2,10 +2,8 @@ import logging
 import os
 import re
 from collections.abc import (
-    Callable,
     Iterable,
     Mapping,
-    Set,
 )
 from functools import cached_property
 from operator import (
@@ -66,6 +64,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 MR_DESCRIPTION_COMMENT_ID = 0
 
 DEFAULT_MAIN_BRANCH = "master"
+MAX_PER_PAGE = 100
 
 
 class MRState:
@@ -135,6 +134,8 @@ class GitLabApi:
             ssl_verify=self.ssl_verify,
             timeout=timeout,
             session=self.session,
+            per_page=MAX_PER_PAGE,
+            pagination="keyset",
         )
         self._auth()
         assert self.gl.user
@@ -267,15 +268,15 @@ class GitLabApi:
         project = self.project if repo_url is None else self.get_project(repo_url)
         if project is None:
             return None
-        if query:
-            members = self.get_items(project.members_all.list, query_parameters=query)
-        else:
-            members = self.get_items(project.members_all.list)
+        members = project.members.list(iterator=True, query_parameters=query or {})
         return [m.username for m in members if m.access_level >= 40]
 
     def get_app_sre_group_users(self) -> list[GroupMember]:
         app_sre_group = self.gl.groups.get("app-sre")
-        return self.get_items(app_sre_group.members.list)
+        return cast(
+            list[GroupMember],
+            app_sre_group.members.list(get_all=True),
+        )
 
     def get_group_if_exists(self, group_name: str) -> Group | None:
         try:
@@ -309,16 +310,12 @@ class GitLabApi:
         """
         return GROUP_BOT_NAME_REGEX.match(username) is not None
 
-    def get_group_members(self, group: Group | None) -> list[GroupMember]:
-        if group is None:
-            logging.error("no group provided")
-            return []
-        else:
-            return [
-                m
-                for m in self.get_items(group.members.list)
-                if not self._is_bot_username(m.username)
-            ]
+    def get_group_members(self, group: Group) -> list[GroupMember]:
+        return [
+            cast(GroupMember, m)
+            for m in group.members.list(iterator=True)
+            if not self._is_bot_username(m.username)
+        ]
 
     def add_project_member(
         self, repo_url: str, user: GroupMember, access: str = "maintainer"
@@ -374,9 +371,9 @@ class GitLabApi:
             case _:
                 raise ValueError(f"Invalid access level: {access}")
 
-    def get_group_id_and_projects(self, group_name: str) -> tuple[str, list[str]]:
+    def get_group_id_and_projects(self, group_name: str) -> tuple[str, set[str]]:
         group = self.gl.groups.get(group_name)
-        return group.id, [p.name for p in self.get_items(group.projects.list)]
+        return group.id, {p.name for p in group.projects.list(iterator=True)}
 
     def get_group(self, group_name: str) -> Group:
         return self.gl.groups.get(group_name)
@@ -401,23 +398,34 @@ class GitLabApi:
         return self.gl.projects.get(project_id)
 
     def get_issues(self, state: str) -> list[ProjectIssue]:
-        return self.get_items(self.project.issues.list, state=state)
+        return cast(
+            list[ProjectIssue],
+            self.project.issues.list(state=state, get_all=True),
+        )
 
     def get_merge_request(self, mr_id: str | int) -> ProjectMergeRequest:
         return self.project.mergerequests.get(mr_id)
 
     def get_merge_requests(self, state: str) -> list[ProjectMergeRequest]:
-        return self.get_items(self.project.mergerequests.list, state=state)
+        return cast(
+            list[ProjectMergeRequest],
+            self.project.mergerequests.list(state=state, get_all=True),
+        )
 
+    @staticmethod
     def get_merge_request_label_events(
-        self, mr: ProjectMergeRequest
+        mr: ProjectMergeRequest,
     ) -> list[ProjectMergeRequestResourceLabelEvent]:
-        return self.get_items(mr.resourcelabelevents.list)
+        return cast(
+            list[ProjectMergeRequestResourceLabelEvent],
+            mr.resourcelabelevents.list(get_all=True),
+        )
 
-    def get_merge_request_pipelines(self, mr: ProjectMergeRequest) -> list[dict]:
+    @staticmethod
+    def get_merge_request_pipelines(mr: ProjectMergeRequest) -> list[dict]:
         # TODO: use typed object in return
         # TODO: use server side order_by
-        items = self.get_items(mr.pipelines.list)
+        items = mr.pipelines.list(iterator=True)
         return sorted(
             [i.asdict() for i in items],
             key=itemgetter("created_at"),
@@ -457,7 +465,7 @@ class GitLabApi:
                 "created_at": merge_request.created_at,
                 "id": MR_DESCRIPTION_COMMENT_ID,
             })
-        for note in GitLabApi.get_items(merge_request.notes.list):
+        for note in merge_request.notes.list(iterator=True):
             if note.system:
                 continue
             comments.append({
@@ -485,8 +493,8 @@ class GitLabApi:
                 self.delete_comment(c["note"])
 
     @retry()
-    def get_project_labels(self) -> Set[str]:
-        return {ln.name for ln in self.get_items(self.project.labels.list)}
+    def get_project_labels(self) -> set[str]:
+        return {label.name for label in self.project.labels.list(iterator=True)}
 
     @staticmethod
     def add_label_to_merge_request(
@@ -552,20 +560,6 @@ class GitLabApi:
     ) -> None:
         merge_request.notes.create({"body": body})
 
-    # TODO: deprecated this method as new support of list(get_all=True)
-    @staticmethod
-    def get_items(method: Callable, **kwargs: Any) -> list:
-        all_items = []
-        page = 1
-        while True:
-            items = method(page=page, per_page=100, **kwargs)
-            all_items.extend(items)
-            if len(items) < 100:
-                break
-            page += 1
-
-        return all_items
-
     def create_label(self, label_text: str, label_color: str) -> None:
         self.project.labels.create({"name": label_text, "color": label_color})
 
@@ -665,11 +659,36 @@ class GitLabApi:
         }
         p.hooks.create(hook)
 
-    def get_repository_tree(self, ref: str = "master") -> list[dict]:
+    def get_repository_tree(
+        self,
+        *,
+        ref: str = "master",
+        recursive: bool = False,
+        project: Project | None = None,
+        path: str = "",
+    ) -> list[dict]:
         """
-        Wrapper around Gitlab.repository_tree() with pagination enabled.
+        Get a list of repository files and directories in a project.
+
+        :param ref: The name of a repository branch or tag or, if not given, the default branch.
+        :param recursive: Boolean value used to get a recursive tree. Default is false.
+        :param project: The project to get the tree from, if None, use the current project
+        :param path: The path inside the repository. Used to get content of subdirectories.
+
+        :return: list of tree objects
         """
-        return self.get_items(self.project.repository_tree, ref=ref, recursive=True)
+        target_project = self.project if project is None else project
+        return cast(
+            list[dict],
+            target_project.repository_tree(
+                ref=ref,
+                path=path,
+                recursive=recursive,
+                pagination="keyset",
+                per_page=MAX_PER_PAGE,
+                get_all=True,
+            ),
+        )
 
     def get_file(self, path: str, ref: str = "master") -> ProjectFile | None:
         """
@@ -762,22 +781,23 @@ class GitLabApi:
         author, assignee = last_assignment[0], last_assignment[1]
         return author in team_usernames and mr.assignee["username"] == assignee
 
-    def last_assignment(self, mr: ProjectMergeRequest) -> tuple[str, str] | None:
+    @staticmethod
+    def last_assignment(mr: ProjectMergeRequest) -> tuple[str, str] | None:
+        """
+        Get the last assignment of a merge request.
+        :param mr: merge request
+        :return: tuple of author name and assignee name or None
+        """
         body_format = "assigned to @"
-        notes = self.get_items(mr.notes.list)
-
-        for note in notes:
-            if not note.system:
-                continue
-            body = note.body
-            if not body.startswith(body_format):
-                continue
-            assignee = body.replace(body_format, "")
-            author = note.author["username"]
-
-            return author, assignee
-
-        return None
+        notes = mr.notes.list(sort="desc", order_by="created_at", iterator=True)
+        return next(
+            (
+                (note.author["username"], note.body.removeprefix(body_format))
+                for note in notes
+                if note.system and note.body.startswith(body_format)
+            ),
+            None,
+        )
 
     def last_comment(
         self, mr: ProjectMergeRequest, exclude_bot: bool = True
@@ -804,4 +824,7 @@ class GitLabApi:
         return response.get("commits", [])
 
     def get_personal_access_tokens(self) -> list[PersonalAccessToken]:
-        return self.get_items(self.gl.personal_access_tokens.list)
+        return cast(
+            list[PersonalAccessToken],
+            self.gl.personal_access_tokens.list(get_all=True),
+        )

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/utils/repo_owners.py

@@ -120,7 +120,7 @@ class RepoOwners:
         aliases = self._get_aliases()
 
         if self._recursive:
-            repo_tree = self._git_cli.get_repository_tree(ref=self._ref)
+            repo_tree = self._git_cli.get_repository_tree(ref=self._ref, recursive=True)
             owner_files = [item for item in repo_tree if item["name"] == "OWNERS"]
         else:
             owner_files = [{"path": "OWNERS"}]

{qontract_reconcile-0.10.2.dev174 → qontract_reconcile-0.10.2.dev176}/reconcile/utils/saasherder/saasherder.py

@@ -798,8 +798,11 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             if not self.gitlab:
                 raise Exception("gitlab is not initialized")
             project = self.gitlab.get_project(url)
-            for item in self.gitlab.get_items(
-                project.repository_tree, path=path.lstrip("/"), ref=commit_sha
+            for item in self.gitlab.get_repository_tree(
+                project=project,
+                path=path.lstrip("/"),
+                ref=commit_sha,
+                recursive=False,
             ):
                 file_contents = project.files.get(
                     file_path=item["path"], ref=commit_sha