qontract-reconcile 0.10.2.dev175__py3-none-any.whl → 0.10.2.dev176__py3-none-any.whl

{qontract_reconcile-0.10.2.dev175.dist-info → qontract_reconcile-0.10.2.dev176.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev175
+Version: 0.10.2.dev176
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev175.dist-info → qontract_reconcile-0.10.2.dev176.dist-info}/RECORD

@@ -32,10 +32,10 @@ reconcile/github_validator.py,sha256=-j17tn3csFVjPMSPL3te48iWVkPZCncRXdeKeLdGjjQ
 reconcile/gitlab_fork_compliance.py,sha256=RbHckzLnE9zkOFHJANzoejEMMbMAivmqJVs3Suvp9lU,4591
 reconcile/gitlab_housekeeping.py,sha256=c31Jtw5t8bnOzUO9jMWF_0DHitPzol93AA7YWBxM5L0,25416
 reconcile/gitlab_labeler.py,sha256=BA2dbXsN9hErUwJl22qcxfeH7XiPCuQ9LN3NddWdnpo,4540
-reconcile/gitlab_members.py,sha256=MUIgYDLeJx2-_vMypyq2Pa17cpKdXATYhtVACS2ghpQ,8297
+reconcile/gitlab_members.py,sha256=yRZOZqwB9_FJ5DWIFEod6hoG0X38z36atInNshAWddI,8263
 reconcile/gitlab_mr_sqs_consumer.py,sha256=i_MDVfA3Uk_TJiNkfEJzhO6_rwR7z3I3dH9oEw686U4,2681
 reconcile/gitlab_owners.py,sha256=nIEsf3QWI3yIw_Bxy5oMaCmszTaNZDwQVaaZZxPgh4g,14447
-reconcile/gitlab_permissions.py,sha256=gSGH6gAdJbPy5Z0rQGUqiNQSHty_tXQ_3Y4OQP_8nFs,8067
+reconcile/gitlab_permissions.py,sha256=kZEdWL0rewP7Odz8amRBPToKxkn0IQn81IoroHGdga4,8101
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=CY7cOj5dzt2se4IOg11VQvGQ-eTvLML5Q42Z9SSgeSk,9463
 reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
@@ -615,8 +615,8 @@ reconcile/utils/external_resource_spec.py,sha256=qeupz4t4trd2uPjlHjf_AFA9Y-EKrMn
 reconcile/utils/external_resources.py,sha256=YzTb0xAcNdmKO326mGQy7BmST56CZcdru4lX7ai_7kw,7579
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=o4p9m8jlzCJDcutl2HErvGLhL6sZ1NB4Aw3zGcQIzso,2427
-reconcile/utils/github_api.py,sha256=y3fxty7FKvfhdzfHgGSaIstL6A_Y2loUcMiyIK5TMDg,2750
-reconcile/utils/gitlab_api.py,sha256=SYSKm5WulWinO7P-ZYy_oerKdfNHHob2V6i7Mfr4oCU,26643
+reconcile/utils/github_api.py,sha256=o4J0ZU1ZSr9808uoorKHv19iae-eLo85yrCZX67p2kw,2822
+reconcile/utils/gitlab_api.py,sha256=5zyC4bYKxwKdPVbRRTXUBaqxgEiQBh52jHbi9G_tBHI,27398
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -653,7 +653,7 @@ reconcile/utils/promtool.py,sha256=xmPBWEApkk0L2qZBAvTxakNXxfTz-tVLPFxGnpsxXnM,2
 reconcile/utils/quay_api.py,sha256=uE_jxcdy3ViHtYFAfwDQuFDaO7Pr6AAPoVnmORbyHio,7822
 reconcile/utils/quay_mirror.py,sha256=dpWCNv5lITwIk6Q9RkmqaQKHNk_JPy27UQEribJ7E-U,1324
 reconcile/utils/raw_github_api.py,sha256=2WKtE8ABYYB9UGOAh9N_kLkksBWL3320Z2_scteZddI,2805
-reconcile/utils/repo_owners.py,sha256=BHrAXxKyvn4qWJwFPWYGTtfgnLmYnWtYFEJGFeD__FE,6573
+reconcile/utils/repo_owners.py,sha256=Xwe1HOcMZe7Pknk47GLZHg5LDpDElmGfmc_x6pAdzsg,6589
 reconcile/utils/rest_api_base.py,sha256=MT7tp6CQO2S5aKfVOzw_hipWg7wAGoOqkm4qurI1hEU,4342
 reconcile/utils/ruamel.py,sha256=FzL4_L0FnMOUZmgThrZSMJs5MTdXwiy-E9MZWfk8bh8,397
 reconcile/utils/secret_reader.py,sha256=MaP56KZaAE35EyYbgAitdm6fUSxdzWeGFSOym9qiZkw,10206
@@ -761,7 +761,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
 reconcile/utils/saasherder/__init__.py,sha256=3U8plqMAPRE1kjwZ5YnIsYsggTf4_gS7flRUEuXVBAs,343
 reconcile/utils/saasherder/interfaces.py,sha256=NEYQspYfyWQhBeJyNCqSFbixi1A4wRVGB7FeNM5BDCk,9141
 reconcile/utils/saasherder/models.py,sha256=JaOz_DEtudJZhiDe90kaBlJkppFufn81V92oK9PHYx0,10208
-reconcile/utils/saasherder/saasherder.py,sha256=ZeYwUSrWbJ0XkmQv92dUGPrhxd5zBKnDEM7_uzRroFE,87067
+reconcile/utils/saasherder/saasherder.py,sha256=crF_hQBeL9TBv_R6SafAze6xutXFSNEX77KaT4XqjF8,87135
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
@@ -807,7 +807,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev175.dist-info/METADATA,sha256=UCbZxjmW8UpEZhxqFSoasTNEH2w_UT9s0PtMCZj4kfU,24627
-qontract_reconcile-0.10.2.dev175.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev175.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev175.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev176.dist-info/METADATA,sha256=VAwixwvzafGc_zGmUcXBP610KqCPyqenHilqpzHXIW8,24627
+qontract_reconcile-0.10.2.dev176.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev176.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev176.dist-info/RECORD,,

reconcile/gitlab_members.py

@@ -68,9 +68,7 @@ def get_current_state(
     """Get current gitlab group members for all managed groups."""
     return {
         g: CurrentStateSpec(
-            members={
-                u.username: u for u in gl.get_group_members(gitlab_groups_map.get(g))
-            },
+            members={u.username: u for u in gl.get_group_members(gitlab_groups_map[g])},
         )
         for g in instance.managed_groups
     }

reconcile/gitlab_permissions.py

@@ -1,12 +1,12 @@
 import itertools
 import logging
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, cast
 
 from gitlab.exceptions import GitlabGetError
 from gitlab.v4.objects import (
-    GroupProject,
     Project,
+    SharedProject,
 )
 from sretoolbox.utils import threaded
 
@@ -51,9 +51,9 @@ class GroupPermissionHandler:
             for project_repo_url in filtered_project_repos
         }
         # get all projects shared with group
-        shared_projects = self.gl.get_items(self.group.shared_projects.list)
+        shared_projects = self.group.shared_projects.list(iterator=True)
         current_state = {
-            project.web_url: self.extract_group_spec(project)
+            project.web_url: self.extract_group_spec(cast(SharedProject, project))
             for project in shared_projects
         }
         self.reconcile(desired_state, current_state)
@@ -61,13 +61,14 @@ class GroupPermissionHandler:
     def filter_group_owned_projects(self, repos: list[str]) -> set[str]:
         # get only the projects that are owned by  group and its sub groups
         query = {"with_shared": False, "include_subgroups": True}
-        group_owned_projects = self.gl.get_items(
-            self.group.projects.list, query_parameters=query
+        group_owned_projects = self.group.projects.list(
+            query_parameters=query,
+            iterator=True,
         )
         group_owned_repo_list = {project.web_url for project in group_owned_projects}
         return set(repos) - group_owned_repo_list
 
-    def extract_group_spec(self, project: GroupProject) -> GroupSpec:
+    def extract_group_spec(self, project: SharedProject) -> GroupSpec:
         return next(
             GroupSpec(
                 group_name=self.group.name,

reconcile/utils/github_api.py

@@ -54,9 +54,14 @@ class GithubRepositoryApi:
         Align with GitLabApi
         """
 
-    def get_repository_tree(self, ref: str = "master") -> list[dict[str, str]]:
+    def get_repository_tree(
+        self,
+        *,
+        ref: str = "master",
+        recursive: bool = False,
+    ) -> list[dict[str, str]]:
         tree_items = []
-        for item in self._repo.get_git_tree(sha=ref, recursive=True).tree:
+        for item in self._repo.get_git_tree(sha=ref, recursive=recursive).tree:
             tree_item = {"path": item.path, "name": Path(item.path).name}
             tree_items.append(tree_item)
         return tree_items

reconcile/utils/gitlab_api.py

@@ -2,10 +2,8 @@ import logging
 import os
 import re
 from collections.abc import (
-    Callable,
     Iterable,
     Mapping,
-    Set,
 )
 from functools import cached_property
 from operator import (
@@ -66,6 +64,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 MR_DESCRIPTION_COMMENT_ID = 0
 
 DEFAULT_MAIN_BRANCH = "master"
+MAX_PER_PAGE = 100
 
 
 class MRState:
@@ -135,6 +134,8 @@ class GitLabApi:
             ssl_verify=self.ssl_verify,
             timeout=timeout,
             session=self.session,
+            per_page=MAX_PER_PAGE,
+            pagination="keyset",
         )
         self._auth()
         assert self.gl.user
@@ -267,15 +268,15 @@ class GitLabApi:
         project = self.project if repo_url is None else self.get_project(repo_url)
         if project is None:
             return None
-        if query:
-            members = self.get_items(project.members_all.list, query_parameters=query)
-        else:
-            members = self.get_items(project.members_all.list)
+        members = project.members.list(iterator=True, query_parameters=query or {})
         return [m.username for m in members if m.access_level >= 40]
 
     def get_app_sre_group_users(self) -> list[GroupMember]:
         app_sre_group = self.gl.groups.get("app-sre")
-        return self.get_items(app_sre_group.members.list)
+        return cast(
+            list[GroupMember],
+            app_sre_group.members.list(get_all=True),
+        )
 
     def get_group_if_exists(self, group_name: str) -> Group | None:
         try:
@@ -309,16 +310,12 @@ class GitLabApi:
         """
         return GROUP_BOT_NAME_REGEX.match(username) is not None
 
-    def get_group_members(self, group: Group | None) -> list[GroupMember]:
-        if group is None:
-            logging.error("no group provided")
-            return []
-        else:
-            return [
-                m
-                for m in self.get_items(group.members.list)
-                if not self._is_bot_username(m.username)
-            ]
+    def get_group_members(self, group: Group) -> list[GroupMember]:
+        return [
+            cast(GroupMember, m)
+            for m in group.members.list(iterator=True)
+            if not self._is_bot_username(m.username)
+        ]
 
     def add_project_member(
         self, repo_url: str, user: GroupMember, access: str = "maintainer"
@@ -374,9 +371,9 @@ class GitLabApi:
             case _:
                 raise ValueError(f"Invalid access level: {access}")
 
-    def get_group_id_and_projects(self, group_name: str) -> tuple[str, list[str]]:
+    def get_group_id_and_projects(self, group_name: str) -> tuple[str, set[str]]:
         group = self.gl.groups.get(group_name)
-        return group.id, [p.name for p in self.get_items(group.projects.list)]
+        return group.id, {p.name for p in group.projects.list(iterator=True)}
 
     def get_group(self, group_name: str) -> Group:
         return self.gl.groups.get(group_name)
@@ -401,23 +398,34 @@ class GitLabApi:
         return self.gl.projects.get(project_id)
 
     def get_issues(self, state: str) -> list[ProjectIssue]:
-        return self.get_items(self.project.issues.list, state=state)
+        return cast(
+            list[ProjectIssue],
+            self.project.issues.list(state=state, get_all=True),
+        )
 
     def get_merge_request(self, mr_id: str | int) -> ProjectMergeRequest:
         return self.project.mergerequests.get(mr_id)
 
     def get_merge_requests(self, state: str) -> list[ProjectMergeRequest]:
-        return self.get_items(self.project.mergerequests.list, state=state)
+        return cast(
+            list[ProjectMergeRequest],
+            self.project.mergerequests.list(state=state, get_all=True),
+        )
 
+    @staticmethod
     def get_merge_request_label_events(
-        self, mr: ProjectMergeRequest
+        mr: ProjectMergeRequest,
     ) -> list[ProjectMergeRequestResourceLabelEvent]:
-        return self.get_items(mr.resourcelabelevents.list)
+        return cast(
+            list[ProjectMergeRequestResourceLabelEvent],
+            mr.resourcelabelevents.list(get_all=True),
+        )
 
-    def get_merge_request_pipelines(self, mr: ProjectMergeRequest) -> list[dict]:
+    @staticmethod
+    def get_merge_request_pipelines(mr: ProjectMergeRequest) -> list[dict]:
         # TODO: use typed object in return
         # TODO: use server side order_by
-        items = self.get_items(mr.pipelines.list)
+        items = mr.pipelines.list(iterator=True)
         return sorted(
             [i.asdict() for i in items],
             key=itemgetter("created_at"),
@@ -457,7 +465,7 @@ class GitLabApi:
                 "created_at": merge_request.created_at,
                 "id": MR_DESCRIPTION_COMMENT_ID,
             })
-        for note in GitLabApi.get_items(merge_request.notes.list):
+        for note in merge_request.notes.list(iterator=True):
             if note.system:
                 continue
             comments.append({
@@ -485,8 +493,8 @@ class GitLabApi:
                 self.delete_comment(c["note"])
 
     @retry()
-    def get_project_labels(self) -> Set[str]:
-        return {ln.name for ln in self.get_items(self.project.labels.list)}
+    def get_project_labels(self) -> set[str]:
+        return {label.name for label in self.project.labels.list(iterator=True)}
 
     @staticmethod
     def add_label_to_merge_request(
@@ -552,20 +560,6 @@ class GitLabApi:
     ) -> None:
         merge_request.notes.create({"body": body})
 
-    # TODO: deprecated this method as new support of list(get_all=True)
-    @staticmethod
-    def get_items(method: Callable, **kwargs: Any) -> list:
-        all_items = []
-        page = 1
-        while True:
-            items = method(page=page, per_page=100, **kwargs)
-            all_items.extend(items)
-            if len(items) < 100:
-                break
-            page += 1
-
-        return all_items
-
     def create_label(self, label_text: str, label_color: str) -> None:
         self.project.labels.create({"name": label_text, "color": label_color})
 
@@ -665,11 +659,36 @@ class GitLabApi:
         }
         p.hooks.create(hook)
 
-    def get_repository_tree(self, ref: str = "master") -> list[dict]:
+    def get_repository_tree(
+        self,
+        *,
+        ref: str = "master",
+        recursive: bool = False,
+        project: Project | None = None,
+        path: str = "",
+    ) -> list[dict]:
         """
-        Wrapper around Gitlab.repository_tree() with pagination enabled.
+        Get a list of repository files and directories in a project.
+
+        :param ref: The name of a repository branch or tag or, if not given, the default branch.
+        :param recursive: Boolean value used to get a recursive tree. Default is false.
+        :param project: The project to get the tree from, if None, use the current project
+        :param path: The path inside the repository. Used to get content of subdirectories.
+
+        :return: list of tree objects
         """
-        return self.get_items(self.project.repository_tree, ref=ref, recursive=True)
+        target_project = self.project if project is None else project
+        return cast(
+            list[dict],
+            target_project.repository_tree(
+                ref=ref,
+                path=path,
+                recursive=recursive,
+                pagination="keyset",
+                per_page=MAX_PER_PAGE,
+                get_all=True,
+            ),
+        )
 
     def get_file(self, path: str, ref: str = "master") -> ProjectFile | None:
         """
@@ -762,22 +781,23 @@ class GitLabApi:
         author, assignee = last_assignment[0], last_assignment[1]
         return author in team_usernames and mr.assignee["username"] == assignee
 
-    def last_assignment(self, mr: ProjectMergeRequest) -> tuple[str, str] | None:
+    @staticmethod
+    def last_assignment(mr: ProjectMergeRequest) -> tuple[str, str] | None:
+        """
+        Get the last assignment of a merge request.
+        :param mr: merge request
+        :return: tuple of author name and assignee name or None
+        """
         body_format = "assigned to @"
-        notes = self.get_items(mr.notes.list)
-
-        for note in notes:
-            if not note.system:
-                continue
-            body = note.body
-            if not body.startswith(body_format):
-                continue
-            assignee = body.replace(body_format, "")
-            author = note.author["username"]
-
-            return author, assignee
-
-        return None
+        notes = mr.notes.list(sort="desc", order_by="created_at", iterator=True)
+        return next(
+            (
+                (note.author["username"], note.body.removeprefix(body_format))
+                for note in notes
+                if note.system and note.body.startswith(body_format)
+            ),
+            None,
+        )
 
     def last_comment(
         self, mr: ProjectMergeRequest, exclude_bot: bool = True
@@ -804,4 +824,7 @@ class GitLabApi:
         return response.get("commits", [])
 
     def get_personal_access_tokens(self) -> list[PersonalAccessToken]:
-        return self.get_items(self.gl.personal_access_tokens.list)
+        return cast(
+            list[PersonalAccessToken],
+            self.gl.personal_access_tokens.list(get_all=True),
+        )

reconcile/utils/repo_owners.py

@@ -120,7 +120,7 @@ class RepoOwners:
         aliases = self._get_aliases()
 
         if self._recursive:
-            repo_tree = self._git_cli.get_repository_tree(ref=self._ref)
+            repo_tree = self._git_cli.get_repository_tree(ref=self._ref, recursive=True)
             owner_files = [item for item in repo_tree if item["name"] == "OWNERS"]
         else:
             owner_files = [{"path": "OWNERS"}]

reconcile/utils/saasherder/saasherder.py

@@ -798,8 +798,11 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             if not self.gitlab:
                 raise Exception("gitlab is not initialized")
             project = self.gitlab.get_project(url)
-            for item in self.gitlab.get_items(
-                project.repository_tree, path=path.lstrip("/"), ref=commit_sha
+            for item in self.gitlab.get_repository_tree(
+                project=project,
+                path=path.lstrip("/"),
+                ref=commit_sha,
+                recursive=False,
             ):
                 file_contents = project.files.get(
                     file_path=item["path"], ref=commit_sha