qontract-reconcile 0.10.2.dev233__tar.gz → 0.10.2.dev234__tar.gz

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev233
+Version: 0.10.2.dev234
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile
@@ -43,7 +43,7 @@ Requires-Dist: pygithub<1.59,>=1.58
 Requires-Dist: pyjwt~=2.7
 Requires-Dist: pyopenssl~=23.0
 Requires-Dist: pypd<1.2.0,>=1.1.0
-Requires-Dist: python-gitlab==5.6.0
+Requires-Dist: python-gitlab==6.0.0
 Requires-Dist: requests-oauthlib~=1.3
 Requires-Dist: requests~=2.32
 Requires-Dist: rich<14.0.0,>=13.3.0

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/pyproject.toml

@@ -45,7 +45,7 @@ dependencies = [
     "pyjwt~=2.7",
     "pyOpenSSL~=23.0",
     "pypd>=1.1.0,<1.2.0",
-    "python-gitlab==5.6.0",
+    "python-gitlab==6.0.0",
     "requests-oauthlib~=1.3",
     "requests~=2.32",
     "rich>=13.3.0,<14.0.0",

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/change_owners/change_owners.py

@@ -382,7 +382,7 @@ def run(
 
             comments = gl.get_merge_request_comments(merge_request)
             good_to_test_approvers = {
-                c["username"] for c in comments if c["body"].strip() == "/good-to-test"
+                c.username for c in comments if c.body.strip() == "/good-to-test"
             }
 
             change_admitted = is_change_admitted(

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/change_owners/decision.py

@@ -1,12 +1,8 @@
-import operator
 from collections import defaultdict
-from collections.abc import (
-    Iterable,
-    Mapping,
-)
+from collections.abc import Iterable
 from dataclasses import dataclass
 from enum import Enum
-from typing import Any
+from operator import attrgetter
 
 from reconcile.change_owners.approver import (
     Approver,
@@ -16,6 +12,7 @@ from reconcile.change_owners.bundle import FileRef
 from reconcile.change_owners.change_types import ChangeTypeContext, DiffCoverage
 from reconcile.change_owners.changes import BundleFileChange
 from reconcile.change_owners.diff import Diff
+from reconcile.utils.gitlab_api import Comment
 
 
 class DecisionCommand(Enum):
@@ -33,12 +30,12 @@ class Decision:
 
 
 def get_approver_decisions_from_mr_comments(
-    comments: Iterable[Mapping[str, Any]],
+    comments: Iterable[Comment],
 ) -> list[Decision]:
     decisions: list[Decision] = []
-    for c in sorted(comments, key=operator.itemgetter("created_at")):
-        commenter = c["username"]
-        comment_body = c.get("body")
+    for c in sorted(comments, key=attrgetter("created_at")):
+        commenter = c.username
+        comment_body = c.body
         for line in comment_body.split("\n") if comment_body else []:
             line = line.strip()
             if line == DecisionCommand.APPROVED.value:

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/gitlab_fork_compliance.py

@@ -77,8 +77,7 @@ class GitlabForkCompliance:
                     continue
                 LOG.info([f"adding {member.username} as maintainer"])
                 user_payload = {"user_id": member.id, "access_level": MAINTAINER_ACCESS}
-                member = self.src.project.members.create(user_payload)
-                member.save()
+                self.src.project.members.create(user_payload)
 
         # Last but not least, we remove the blocked label, in case
         # it is set

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/gitlab_housekeeping.py

@@ -14,10 +14,12 @@ from operator import itemgetter
 from typing import Any, cast
 
 import gitlab
+from gitlab.const import PipelineStatus
 from gitlab.v4.objects import (
     ProjectCommit,
     ProjectIssue,
     ProjectMergeRequest,
+    ProjectMergeRequestPipeline,
 )
 from prometheus_client import (
     Counter,
@@ -133,11 +135,16 @@ def _calculate_time_since_approval(approved_at: str) -> float:
 
 
 def get_timed_out_pipelines(
-    pipelines: list[dict], pipeline_timeout: int = 60
-) -> list[dict]:
+    pipelines: list[ProjectMergeRequestPipeline],
+    pipeline_timeout: int = 60,
+) -> list[ProjectMergeRequestPipeline]:
     now = datetime.utcnow()
 
-    pending_pipelines = [p for p in pipelines if p["status"] in {"pending", "running"}]
+    pending_pipelines = [
+        p
+        for p in pipelines
+        if p.status in {PipelineStatus.PENDING, PipelineStatus.RUNNING}
+    ]
 
     if not pending_pipelines:
         return []
@@ -145,7 +152,7 @@ def get_timed_out_pipelines(
     timed_out_pipelines = []
 
     for p in pending_pipelines:
-        update_time = datetime.strptime(p["updated_at"], DATE_FORMAT)
+        update_time = datetime.strptime(p.updated_at, DATE_FORMAT)
 
         elapsed = (now - update_time).total_seconds()
 
@@ -160,20 +167,19 @@ def clean_pipelines(
     dry_run: bool,
     gl: GitLabApi,
     fork_project_id: int,
-    pipelines: list[dict],
+    pipelines: list[ProjectMergeRequestPipeline],
 ) -> None:
     if not dry_run:
         gl_piplelines = gl.get_project_by_id(fork_project_id).pipelines
 
     for p in pipelines:
-        logging.info(["canceling", p["web_url"]])
+        logging.info(["canceling", p.web_url])
         if not dry_run:
             try:
-                gl_piplelines.get(p["id"]).cancel()
+                gl_piplelines.get(p.id, lazy=True).cancel()
             except gitlab.exceptions.GitlabPipelineCancelError as err:
                 logging.error(
-                    f"unable to cancel {p['web_url']} - "
-                    f"error message {err.error_message}"
+                    f"unable to cancel {p.web_url} - error message {err.error_message}"
                 )
 
 
@@ -188,9 +194,9 @@ def verify_on_demand_tests(
     Check if MR has passed all necessary test jobs and add comments to indicate test results.
     """
     pipelines = gl.get_merge_request_pipelines(mr)
-    running_pipelines = [p for p in pipelines if p["status"] == "running"]
+    running_pipelines = [p for p in pipelines if p.status == PipelineStatus.RUNNING]
     if running_pipelines:
-        # wait for pipelines complate
+        # wait for pipelines completion
         return False
 
     commit = next(mr.commits())
@@ -510,7 +516,9 @@ def rebase_merge_requests(
                 continue
             # possible statuses:
             # running, pending, success, failed, canceled, skipped
-            running_pipelines = [p for p in pipelines if p["status"] == "running"]
+            running_pipelines = [
+                p for p in pipelines if p.status == PipelineStatus.RUNNING
+            ]
             if running_pipelines:
                 continue
 
@@ -591,7 +599,9 @@ def merge_merge_requests(
         if wait_for_pipeline:
             # possible statuses:
             # running, pending, success, failed, canceled, skipped
-            running_pipelines = [p for p in pipelines if p["status"] == "running"]
+            running_pipelines = [
+                p for p in pipelines if p.status == PipelineStatus.RUNNING
+            ]
             if running_pipelines:
                 if insist:
                     # This raise causes the method to restart due to the usage of
@@ -606,8 +616,8 @@ def merge_merge_requests(
                     )
                 continue
 
-        last_pipeline_result = pipelines[0]["status"]
-        if last_pipeline_result != "success":
+        last_pipeline_result = pipelines[0].status
+        if last_pipeline_result != PipelineStatus.SUCCESS:
             continue
 
         logging.info(["merge", gl.project.name, mr.iid])

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/gitlab_members.py

@@ -216,7 +216,7 @@ def reconcile_gitlab_members(
                 gl.get_access_level_string(gitlab_user.access_level),
             ])
             if not dry_run:
-                gl.add_group_member(group, gitlab_user)
+                gl.add_group_member(group, gitlab_user.user, gitlab_user.access_level)
         for key, group_member in diff_data.delete.items():
             logging.info([
                 key,

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/gitlab_owners.py

@@ -89,19 +89,19 @@ class MRApproval:
         comments = self.gitlab.get_merge_request_comments(self.mr)
         for comment in comments:
             # Only interested in '/lgtm' comments
-            if comment["body"] != "/lgtm":
+            if comment.body != "/lgtm":
                 continue
 
             # Only interested in comments created after the top commit
             # creation time
-            comment_created_at = dateparser.parse(comment["created_at"])
+            comment_created_at = dateparser.parse(comment.created_at)
             if (
                 comment_created_at < self.top_commit_created_at
                 and not self.persistent_lgtm
             ):
                 continue
 
-            lgtms.append(comment["username"])
+            lgtms.append(comment.username)
         return lgtms
 
     def expand_groups(self, owners: list[str]) -> set[str]:
@@ -173,19 +173,22 @@ class MRApproval:
         comments = self.gitlab.get_merge_request_comments(self.mr)
         for comment in comments:
             # Only interested on our own comments
-            if comment["username"] != self.gitlab.user.username:
+            if comment.username != self.gitlab.user.username:
                 continue
 
             # Ignoring non-approval comments
-            body = comment["body"]
+            body = comment.body
             if not body.startswith(COMMENT_PREFIX):
                 continue
 
             # If the comment was created before the last commit,
             # it means we had a push after the comment. In this case,
             # we delete the comment and move on.
-            comment_created_at = dateparser.parse(comment["created_at"])
-            if comment_created_at < self.top_commit_created_at:
+            comment_created_at = dateparser.parse(comment.created_at)
+            if (
+                comment_created_at < self.top_commit_created_at
+                and comment.note is not None
+            ):
                 # Deleting stale comments
                 _LOG.info([
                     f"Project:{self.gitlab.project.id} "
@@ -193,7 +196,7 @@ class MRApproval:
                     f"- removing stale comment"
                 ])
                 if not self.dry_run:
-                    self.gitlab.delete_comment(comment["note"])
+                    self.gitlab.delete_comment(comment.note)
                 continue
 
             # At this point, we've found an approval comment comment

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/utils/gitlab_api.py

@@ -7,16 +7,12 @@ from collections.abc import (
     Iterable,
     Mapping,
 )
+from dataclasses import dataclass
 from functools import cached_property
-from operator import (
-    attrgetter,
-    itemgetter,
-)
+from operator import attrgetter
 from typing import (
     Any,
-    Protocol,
     Self,
-    TypedDict,
     cast,
 )
 from urllib.parse import urlparse
@@ -47,6 +43,7 @@ from gitlab.v4.objects import (
     ProjectMergeRequest,
     ProjectMergeRequestManager,
     ProjectMergeRequestNote,
+    ProjectMergeRequestPipeline,
     ProjectMergeRequestResourceLabelEvent,
     User,
 )
@@ -97,15 +94,19 @@ class MRStatus:
 GROUP_BOT_NAME_REGEX = re.compile(r"group_.+_bot_.+")
 
 
-class GLGroupMember(TypedDict):
-    id: str
-    user: str
-    access_level: str
+@dataclass(frozen=True)
+class Assignment:
+    author: str
+    assignee: str
 
 
-class GitlabUser(Protocol):
-    user: str
-    access_level: int
+@dataclass(frozen=True)
+class Comment:
+    id: int
+    username: str
+    body: str
+    created_at: str
+    note: ProjectMergeRequestNote | None = None
 
 
 class GitLabApi:
@@ -254,7 +255,7 @@ class GitLabApi:
             "remove_source_branch": str(remove_source_branch),
             "labels": labels,
         }
-        return cast(ProjectMergeRequest, self.project.mergerequests.create(data))
+        return self.project.mergerequests.create(data)
 
     def mr_exists(self, title: str) -> bool:
         mrs = self.get_merge_requests(state=MRState.OPENED)
@@ -274,10 +275,7 @@ class GitLabApi:
 
     def get_app_sre_group_users(self) -> list[GroupMember]:
         app_sre_group = self.gl.groups.get("app-sre")
-        return cast(
-            list[GroupMember],
-            app_sre_group.members.list(get_all=True),
-        )
+        return app_sre_group.members.list(get_all=True)
 
     def get_group_if_exists(self, group_name: str) -> Group | None:
         try:
@@ -312,7 +310,7 @@ class GitLabApi:
 
     def get_group_members(self, group: Group) -> list[GroupMember]:
         return [
-            cast(GroupMember, m)
+            m
             for m in group.members.list(iterator=True)
             if not self._is_bot_username(m.username)
         ]
@@ -331,17 +329,17 @@ class GitLabApi:
             member.access_level = access_level
             member.save()
 
-    def add_group_member(self, group: Group, user: GitlabUser) -> None:
-        gitlab_user = self.get_user(user.user)
+    def add_group_member(self, group: Group, username: str, access_level: int) -> None:
+        gitlab_user = self.get_user(username)
         if gitlab_user is not None:
             try:
                 group.members.create({
                     "user_id": gitlab_user.id,
-                    "access_level": user.access_level,
+                    "access_level": access_level,
                 })
             except GitlabCreateError:
-                member = group.members.get(user.user)
-                member.access_level = user.access_level
+                member = group.members.get(gitlab_user.id)
+                member.access_level = access_level
                 member.save()
 
     def remove_group_member(self, group: Group, user_id: str) -> None:
@@ -398,37 +396,27 @@ class GitLabApi:
         return self.gl.projects.get(project_id)
 
     def get_issues(self, state: str) -> list[ProjectIssue]:
-        return cast(
-            list[ProjectIssue],
-            self.project.issues.list(state=state, get_all=True),
-        )
+        return self.project.issues.list(state=state, get_all=True)
 
     def get_merge_request(self, mr_id: str | int) -> ProjectMergeRequest:
         return self.project.mergerequests.get(mr_id)
 
     def get_merge_requests(self, state: str) -> list[ProjectMergeRequest]:
-        return cast(
-            list[ProjectMergeRequest],
-            self.project.mergerequests.list(state=state, get_all=True),
-        )
+        return self.project.mergerequests.list(state=state, get_all=True)
 
     @staticmethod
     def get_merge_request_label_events(
         mr: ProjectMergeRequest,
     ) -> list[ProjectMergeRequestResourceLabelEvent]:
-        return cast(
-            list[ProjectMergeRequestResourceLabelEvent],
-            mr.resourcelabelevents.list(get_all=True),
-        )
+        return mr.resourcelabelevents.list(get_all=True)
 
     @staticmethod
-    def get_merge_request_pipelines(mr: ProjectMergeRequest) -> list[dict]:
-        # TODO: use typed object in return
-        # TODO: use server side order_by
-        items = mr.pipelines.list(iterator=True)
+    def get_merge_request_pipelines(
+        mr: ProjectMergeRequest,
+    ) -> list[ProjectMergeRequestPipeline]:
         return sorted(
-            [i.asdict() for i in items],
-            key=itemgetter("created_at"),
+            mr.pipelines.list(iterator=True),
+            key=attrgetter("created_at"),
             reverse=True,
         )
 
@@ -456,25 +444,28 @@ class GitLabApi:
     def get_merge_request_comments(
         merge_request: ProjectMergeRequest,
         include_description: bool = False,
-    ) -> list[dict[str, Any]]:
+    ) -> list[Comment]:
         comments = []
         if include_description:
-            comments.append({
-                "username": merge_request.author["username"],
-                "body": merge_request.description,
-                "created_at": merge_request.created_at,
-                "id": MR_DESCRIPTION_COMMENT_ID,
-            })
-        for note in merge_request.notes.list(iterator=True):
-            if note.system:
-                continue
-            comments.append({
-                "username": note.author["username"],
-                "body": note.body,
-                "created_at": note.created_at,
-                "id": note.id,
-                "note": note,
-            })
+            comments.append(
+                Comment(
+                    id=MR_DESCRIPTION_COMMENT_ID,
+                    username=merge_request.author["username"],
+                    body=merge_request.description or "",
+                    created_at=merge_request.created_at,
+                )
+            )
+        comments.extend(
+            Comment(
+                id=note.id,
+                username=note.author["username"],
+                body=note.body or "",
+                created_at=note.created_at,
+                note=note,
+            )
+            for note in merge_request.notes.list(iterator=True)
+            if not note.system
+        )
         return comments
 
     @staticmethod
@@ -488,9 +479,12 @@ class GitLabApi:
     ) -> None:
         comments = self.get_merge_request_comments(merge_request)
         for c in comments:
-            body = c["body"] or ""
-            if c["username"] == self.user.username and body.startswith(startswith):
-                self.delete_comment(c["note"])
+            if (
+                c.username == self.user.username
+                and c.body.startswith(startswith)
+                and c.note is not None
+            ):
+                self.delete_comment(c.note)
 
     @retry()
     def get_project_labels(self) -> set[str]:
@@ -631,7 +625,7 @@ class GitLabApi:
         item.save()
 
     def get_user(self, username: str) -> User | None:
-        user = cast(list[User], self.gl.users.list(search=username, page=1, per_page=1))
+        user = self.gl.users.list(search=username, page=1, per_page=1)
         if not user:
             logging.error(f"{username} user not found")
             return None
@@ -732,25 +726,22 @@ class GitLabApi:
         self.create_branch("production", "master")
 
     def is_last_action_by_team(
-        self, mr: ProjectMergeRequest, team_usernames: list[str], hold_labels: list[str]
+        self, mr: ProjectMergeRequest, team_usernames: set[str], hold_labels: list[str]
     ) -> bool:
         # what is the time of the last app-sre response?
         last_action_by_team = None
         # comments
         comments = self.get_merge_request_comments(mr)
-        comments.sort(key=itemgetter("created_at"), reverse=True)
+        comments.sort(key=attrgetter("created_at"), reverse=True)
         for comment in comments:
-            username = comment["username"]
+            username = comment.username
             if username == self.user.username:
                 continue
             if username in team_usernames:
-                last_action_by_team = comment["created_at"]
+                last_action_by_team = comment.created_at
                 break
         # labels
-        label_events = cast(
-            list[ProjectMergeRequestResourceLabelEvent],
-            mr.resourcelabelevents.list(get_all=True),
-        )
+        label_events = mr.resourcelabelevents.list(get_all=True)
         for label in reversed(label_events):
             if label.action == "add" and label.label["name"] in hold_labels:
                 username = label.user["username"]
@@ -774,11 +765,11 @@ class GitLabApi:
             break
         # comments
         for comment in comments:
-            username = comment["username"]
+            username = comment.username
             if username == self.user.username:
                 continue
             if username not in team_usernames:
-                last_action_not_by_team = comment["created_at"]
+                last_action_not_by_team = comment.created_at
                 break
 
         if not last_action_not_by_team:
@@ -787,19 +778,20 @@ class GitLabApi:
         return last_action_not_by_team < last_action_by_team
 
     def is_assigned_by_team(
-        self, mr: ProjectMergeRequest, team_usernames: list[str]
+        self, mr: ProjectMergeRequest, team_usernames: set[str]
     ) -> bool:
         if not mr.assignee:
             return False
         last_assignment = self.last_assignment(mr)
-        if not last_assignment:
+        if last_assignment is None:
             return False
-
-        author, assignee = last_assignment[0], last_assignment[1]
-        return author in team_usernames and mr.assignee["username"] == assignee
+        return (
+            last_assignment.author in team_usernames
+            and mr.assignee["username"] == last_assignment.assignee
+        )
 
     @staticmethod
-    def last_assignment(mr: ProjectMergeRequest) -> tuple[str, str] | None:
+    def last_assignment(mr: ProjectMergeRequest) -> Assignment | None:
         """
         Get the last assignment of a merge request.
         :param mr: merge request
@@ -809,7 +801,10 @@ class GitLabApi:
         notes = mr.notes.list(sort="desc", order_by="created_at", iterator=True)
         return next(
             (
-                (note.author["username"], note.body.removeprefix(body_format))
+                Assignment(
+                    author=note.author["username"],
+                    assignee=note.body.removeprefix(body_format),
+                )
                 for note in notes
                 if note.system and note.body.startswith(body_format)
             ),
@@ -818,15 +813,17 @@ class GitLabApi:
 
     def last_comment(
         self, mr: ProjectMergeRequest, exclude_bot: bool = True
-    ) -> dict[str, Any] | None:
+    ) -> Comment | None:
         comments = self.get_merge_request_comments(mr)
-        comments.sort(key=itemgetter("created_at"), reverse=True)
-        for comment in comments:
-            username = comment["username"]
-            if username == self.user.username and exclude_bot:
-                continue
-            return comment
-        return None
+        comments.sort(key=attrgetter("created_at"), reverse=True)
+        return next(
+            (
+                comment
+                for comment in comments
+                if not (exclude_bot and comment.username == self.user.username)
+            ),
+            None,
+        )
 
     def get_commit_sha(self, ref: str, repo_url: str) -> str:
         project = self.get_project(repo_url)
@@ -841,10 +838,7 @@ class GitLabApi:
         return response.get("commits", [])
 
     def get_personal_access_tokens(self) -> list[PersonalAccessToken]:
-        return cast(
-            list[PersonalAccessToken],
-            self.gl.personal_access_tokens.list(get_all=True),
-        )
+        return self.gl.personal_access_tokens.list(get_all=True)
 
     @staticmethod
     def get_directory_contents(

{qontract_reconcile-0.10.2.dev233 → qontract_reconcile-0.10.2.dev234}/reconcile/utils/vcs.py

@@ -9,6 +9,7 @@ from enum import Enum
 from typing import Literal
 from urllib.parse import urlparse
 
+from gitlab.const import PipelineStatus
 from gitlab.v4.objects import ProjectMergeRequest
 
 from reconcile.typed_queries.github_orgs import GithubOrgV1
@@ -154,14 +155,13 @@ class VCS:
         pipelines = self._gitlab_instance.get_merge_request_pipelines(mr)
         if not pipelines:
             return MRCheckStatus.NONE
-        # available status codes https://docs.gitlab.com/ee/api/pipelines.html
-        last_pipeline_result = pipelines[0]["status"]
+        last_pipeline_result = pipelines[0].status
         match last_pipeline_result:
-            case "success":
+            case PipelineStatus.SUCCESS:
                 return MRCheckStatus.SUCCESS
-            case "running":
+            case PipelineStatus.RUNNING:
                 return MRCheckStatus.RUNNING
-            case "failed":
+            case PipelineStatus.FAILED:
                 return MRCheckStatus.FAILED
             case _:
                 # Lets assume all other states as non-present