qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/gitlab_housekeeping.py

@@ -1,15 +1,17 @@
 import logging
-from collections.abc import Iterable
+from collections.abc import (
+    Iterable,
+    Set,
+)
+from contextlib import suppress
+from dataclasses import dataclass
 from datetime import (
+    UTC,
     datetime,
     timedelta,
 )
 from operator import itemgetter
-from typing import (
-    Any,
-    Optional,
-    Union,
-)
+from typing import Any
 
 import gitlab
 from gitlab.v4.objects import (
@@ -40,10 +42,13 @@ from reconcile.utils.mr.labels import (
     DO_NOT_MERGE_PENDING_REVIEW,
     HOLD,
     LGTM,
+    NEEDS_REBASE,
+    ONBOARDING,
     SAAS_FILE_UPDATE,
     SELF_SERVICEABLE,
     prioritized_approval_label,
 )
+from reconcile.utils.sharding import is_in_shard
 
 MERGE_LABELS_PRIORITY = [
     prioritized_approval_label(p.value) for p in ChangeTypePriority
@@ -59,16 +64,18 @@ HOLD_LABELS = [
     HOLD,
     DO_NOT_MERGE_HOLD,
     DO_NOT_MERGE_PENDING_REVIEW,
+    NEEDS_REBASE,
 ]
 
 QONTRACT_INTEGRATION = "gitlab-housekeeping"
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+EXPIRATION_DATE_FORMAT = "%Y-%m-%d"
 
 
 merged_merge_requests = Counter(
     name="qontract_reconcile_merged_merge_requests",
     documentation="Number of merge requests that have been successfully merged in a repository",
-    labelnames=["project_id"],
+    labelnames=["project_id", "self_service", "auto_merge", "app_sre", "onboarding"],
 )
 
 rebased_merge_requests = Counter(
@@ -80,7 +87,7 @@ rebased_merge_requests = Counter(
 time_to_merge = Histogram(
     name="qontract_reconcile_time_to_merge_merge_request_minutes",
     documentation="The number of minutes it takes from when a merge request is mergeable until it is actually merged. This is an indicator of how busy the merge queue is.",
-    labelnames=["project_id"],
+    labelnames=["project_id", "priority"],
     buckets=(5.0, 10.0, 20.0, 40.0, 60.0, float("inf")),
 )
 
@@ -90,11 +97,24 @@ merge_requests_waiting = Gauge(
     labelnames=["project_id"],
 )
 
+gitlab_token_expiration = Gauge(
+    name="qontract_reconcile_gitlab_token_expiration_days",
+    documentation="Time until personal access tokens expire",
+    labelnames=["name"],
+)
+
 
 class InsistOnPipelineError(Exception):
     """Exception used to retry a merge when the pipeline isn't yet complete."""
 
 
+@dataclass
+class ReloadToggle:
+    """A class to toggle the reload of merge requests."""
+
+    reload: bool = False
+
+
 def _log_exception(ex: Exception) -> None:
     logging.info("Retrying - %s: %s", type(ex).__name__, ex)
 
@@ -115,7 +135,7 @@ def get_timed_out_pipelines(
 ) -> list[dict]:
     now = datetime.utcnow()
 
-    pending_pipelines = [p for p in pipelines if p["status"] in ["pending", "running"]]
+    pending_pipelines = [p for p in pipelines if p["status"] in {"pending", "running"}]
 
     if not pending_pipelines:
         return []
@@ -164,33 +184,40 @@ def close_item(
     gl: GitLabApi,
     enable_closing: bool,
     item_type: str,
-    item: Union[ProjectIssue, ProjectMergeRequest],
+    item: ProjectIssue | ProjectMergeRequest,
 ):
-    logging.info(["close_item", gl.project.name, item_type, item.attributes.get("iid")])
     if enable_closing:
+        logging.info([
+            "close_item",
+            gl.project.name,
+            item_type,
+            item.attributes.get("iid"),
+        ])
         if not dry_run:
             gl.close(item)
     else:
-        logging.debug(
-            "'close_item' action is not enabled. "
-            + "Please run the integration manually "
-            + "with the '--enable-deletion' flag."
-        )
+        logging.debug([
+            "'enable_closing' is not enabled to close item",
+            gl.project.name,
+            item_type,
+            item.attributes.get("iid"),
+        ])
 
 
-def handle_stale_items(dry_run, gl, days_interval, enable_closing, item_type):
+def handle_stale_items(
+    dry_run,
+    gl,
+    days_interval,
+    enable_closing,
+    items,
+    item_type,
+):
     LABEL = "stale"
 
-    if item_type == "issue":
-        items = gl.get_issues(state=MRState.OPENED)
-    elif item_type == "merge-request":
-        items = gl.get_merge_requests(state=MRState.OPENED)
-
     now = datetime.utcnow()
     for item in items:
         item_iid = item.attributes.get("iid")
-        item_labels = get_labels(item, gl)
-        if AUTO_MERGE in item_labels:
+        if AUTO_MERGE in item.labels:
             if item.merge_status == MRStatus.UNCHECKED:
                 # this call triggers a status recheck
                 item = gl.get_merge_request(item_iid)
@@ -202,23 +229,24 @@ def handle_stale_items(dry_run, gl, days_interval, enable_closing, item_type):
         current_interval = now.date() - update_date.date()
         if current_interval > timedelta(days=days_interval):
             # if item does not have 'stale' label - add it
-            if LABEL not in item_labels:
+            if LABEL not in item.labels:
                 logging.info(["add_label", gl.project.name, item_type, item_iid, LABEL])
                 if not dry_run:
-                    gl.add_label(item, item_type, LABEL)
+                    gl.add_label_with_note(item, LABEL)
             # if item has 'stale' label - close it
             else:
                 close_item(dry_run, gl, enable_closing, item_type, item)
         # if item is under days_interval
         else:
-            if LABEL not in item_labels:
+            if LABEL not in item.labels:
                 continue
 
             # if item has 'stale' label - check the notes
+            # TODO: add request count metrics and maybe server side filter to reduce requests
             cancel_notes = [
                 n
-                for n in item.notes.list()
-                if n.attributes.get("body") == "/{} cancel".format(LABEL)
+                for n in item.notes.list(iterator=True)
+                if n.attributes.get("body") == f"/{LABEL} cancel"
             ]
             if not cancel_notes:
                 continue
@@ -232,11 +260,15 @@ def handle_stale_items(dry_run, gl, days_interval, enable_closing, item_type):
             # days_interval - remove 'stale' label
             current_interval = now.date() - latest_cancel_note_date.date()
             if current_interval <= timedelta(days=days_interval):
-                logging.info(
-                    ["remove_label", gl.project.name, item_type, item_iid, LABEL]
-                )
+                logging.info([
+                    "remove_label",
+                    gl.project.name,
+                    item_type,
+                    item_iid,
+                    LABEL,
+                ])
                 if not dry_run:
-                    gl.remove_label(item, item_type, LABEL)
+                    gl.remove_label(item, LABEL)
 
 
 def is_good_to_merge(labels):
@@ -247,45 +279,48 @@ def is_good_to_merge(labels):
 
 def is_rebased(mr, gl: GitLabApi) -> bool:
     target_branch = mr.target_branch
-    head = gl.project.commits.list(ref_name=target_branch)[0].id
+    head = gl.project.commits.list(
+        ref_name=target_branch,
+        per_page=1,
+        page=1,
+    )[0].id
     result = gl.project.repository_compare(mr.sha, head)
     return len(result["commits"]) == 0
 
 
-def get_labels(mr: ProjectMergeRequest, gl: GitLabApi) -> list[str]:
-    """
-    This function used to contain logic for checking if labels were empty and calling
-    gl.get_merge_request_labels() if they were missing because there were reports of the
-    label attribute being empty sometimes when it shouldn't be. This was an expensive
-    approach, increasing the runtime of the integration by something around 20-30%.
-    Through investigation in APPSRE-6653 it was determined this no longer appears to be
-    an issue.
-
-    This is being left to continue to abstract the way that labels are pulled in case
-    this becomes an issue again in the future.
-    """
-    return mr.attributes.get("labels")
-
-
 def get_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
-    users_allowed_to_label: Optional[Iterable[str]] = None,
+    users_allowed_to_label: Iterable[str] | None = None,
 ) -> list[dict[str, Any]]:
     mrs = gl.get_merge_requests(state=MRState.OPENED)
+    return preprocess_merge_requests(
+        dry_run=dry_run,
+        gl=gl,
+        project_merge_requests=mrs,
+        users_allowed_to_label=users_allowed_to_label,
+    )
+
+
+def preprocess_merge_requests(
+    dry_run: bool,
+    gl: GitLabApi,
+    project_merge_requests: list[ProjectMergeRequest],
+    users_allowed_to_label: Iterable[str] | None = None,
+) -> list[dict[str, Any]]:
     results = []
-    for mr in mrs:
-        if mr.merge_status in [
+    for mr in project_merge_requests:
+        if mr.merge_status in {
             MRStatus.CANNOT_BE_MERGED,
             MRStatus.CANNOT_BE_MERGED_RECHECK,
-        ]:
+        }:
             continue
-        if mr.work_in_progress:
+        if mr.draft:
             continue
         if len(mr.commits()) == 0:
             continue
 
-        labels = get_labels(mr, gl)
+        labels = set(mr.labels)
         if not labels:
             continue
 
@@ -297,7 +332,7 @@ def get_merge_requests(
                 + "suitable for self serviceable MRs. removing 'lgtm' label"
             )
             if not dry_run:
-                gl.remove_label_from_merge_request(mr.iid, LGTM)
+                gl.remove_label(mr, LGTM)
             continue
 
         label_events = gl.get_merge_request_label_events(mr)
@@ -326,22 +361,22 @@ def get_merge_requests(
                     approved_at = label.created_at
                     approved_by = added_by
 
-        for bad_label in labels_by_unauthorized_users - labels_by_authorized_users:
-            if bad_label not in labels:
-                continue
+        bad_labels = (
+            labels_by_unauthorized_users - labels_by_authorized_users
+        ) & labels
+        for bad_label in bad_labels:
             logging.warning(
                 f"[{gl.project.name}/{mr.iid}] someone added a label who "
                 f"isn't allowed. removing label {bad_label}"
             )
-            # Remove bad_label from the cached labels list. Otherwise, we may face a caching bug
-            labels.remove(bad_label)
-            if not dry_run:
-                gl.remove_label_from_merge_request(mr.iid, bad_label)
+        if bad_labels and not dry_run:
+            gl.remove_labels(mr, bad_labels)
 
+        labels = set(mr.labels)
         if not is_good_to_merge(labels):
             continue
 
-        label_priotiry = min(
+        label_priority = min(
             MERGE_LABELS_PRIORITY.index(merge_label)
             for merge_label in MERGE_LABELS_PRIORITY
             if merge_label in labels
@@ -349,8 +384,8 @@ def get_merge_requests(
 
         item = {
             "mr": mr,
-            "labels": labels,
-            "label_priority": label_priotiry,
+            "label_priority": label_priority,
+            "priority": f"{label_priority} - {MERGE_LABELS_PRIORITY[label_priority]}",
             "approved_at": approved_at,
             "approved_by": approved_by,
         }
@@ -410,16 +445,14 @@ def rebase_merge_requests(
                     rebases += 1
                     rebased_merge_requests.labels(mr.target_project_id).inc()
             except gitlab.exceptions.GitlabMRRebaseError as e:
-                logging.error("unable to rebase {}: {}".format(mr.iid, e))
+                logging.error(f"unable to rebase {mr.iid}: {e}")
         else:
-            logging.info(
-                [
-                    "rebase",
-                    gl.project.name,
-                    mr.iid,
-                    "rebase limit reached for this reconcile loop. will try next time",
-                ]
-            )
+            logging.info([
+                "rebase",
+                gl.project.name,
+                mr.iid,
+                "rebase limit reached for this reconcile loop. will try next time",
+            ])
 
 
 # TODO: this retry is catching all exceptions, which isn't good. _log_exceptions is
@@ -432,9 +465,12 @@ def rebase_merge_requests(
 @retry(max_attempts=10, hook=_log_exception)
 def merge_merge_requests(
     dry_run,
-    gl,
+    gl: GitLabApi,
+    project_merge_requests: list[ProjectMergeRequest],
+    reload_toggle: ReloadToggle,
     merge_limit,
     rebase,
+    app_sre_usernames: Set[str],
     pipeline_timeout=None,
     insist=False,
     wait_for_pipeline=False,
@@ -443,15 +479,15 @@ def merge_merge_requests(
     users_allowed_to_label=None,
 ):
     merges = 0
-    mrs = get_merge_requests(dry_run, gl, users_allowed_to_label)
-    merge_requests: list[ProjectMergeRequest] = [item["mr"] for item in mrs]
-    merge_request_approved_at: dict[ProjectMergeRequest, str] = {
-        item["mr"]: item["approved_at"] for item in mrs
-    }
-
+    if reload_toggle.reload:
+        project_merge_requests = gl.get_merge_requests(state=MRState.OPENED)
+    merge_requests = preprocess_merge_requests(
+        dry_run, gl, project_merge_requests, users_allowed_to_label
+    )
     merge_requests_waiting.labels(gl.project.id).set(len(merge_requests))
 
-    for mr in merge_requests:
+    for merge_request in merge_requests:
+        mr: ProjectMergeRequest = merge_request["mr"]
         if rebase and not is_rebased(mr, gl):
             continue
 
@@ -483,8 +519,9 @@ def merge_merge_requests(
                     # for now is being considered a feature because a higher priority
                     # merge request could have become available since we've been waiting
                     # for this pipeline to complete.
+                    reload_toggle.reload = True
                     raise InsistOnPipelineError(
-                        f"Pipelines for merge request have not completed yet: {mr.iid}"
+                        f"Pipelines for merge request in project '{gl.project.name}' have not completed yet: {mr.iid}"
                     )
                 continue
 
@@ -496,15 +533,42 @@ def merge_merge_requests(
         if not dry_run and merges < merge_limit:
             try:
                 mr.merge()
-                merged_merge_requests.labels(mr.target_project_id).inc()
-                time_to_merge.labels(mr.target_project_id).observe(
-                    _calculate_time_since_approval(merge_request_approved_at[mr])
-                )
+                labels = mr.labels
+                merged_merge_requests.labels(
+                    project_id=mr.target_project_id,
+                    self_service=SELF_SERVICEABLE in labels,
+                    auto_merge=AUTO_MERGE in labels,
+                    app_sre=mr.author["username"] in app_sre_usernames,
+                    onboarding=ONBOARDING in labels,
+                ).inc()
+                time_to_merge.labels(
+                    project_id=mr.target_project_id, priority=merge_request["priority"]
+                ).observe(_calculate_time_since_approval(merge_request["approved_at"]))
                 if rebase:
+                    # other merge requests need to be rebased first after this one merged, so terminate
                     return
                 merges += 1
             except gitlab.exceptions.GitlabMRClosedError as e:
-                logging.error("unable to merge {}: {}".format(mr.iid, e))
+                logging.error(f"unable to merge {mr.iid}: {e}")
+
+
+def get_app_sre_usernames(gl: GitLabApi) -> set[str]:
+    return {u.username for u in gl.get_app_sre_group_users()}
+
+
+def publish_access_token_expiration_metrics(gl: GitLabApi) -> None:
+    pats = gl.get_personal_access_tokens()
+
+    for pat in pats:
+        if pat.active:
+            expiration_date = datetime.strptime(pat.expires_at, EXPIRATION_DATE_FORMAT)
+            days_until_expiration = expiration_date.date() - datetime.now(UTC).date()
+            gitlab_token_expiration.labels(pat.name).set(days_until_expiration.days)
+        else:
+            with suppress(KeyError, ValueError):
+                # there's no publicly exposed method to determine if a label exists for a gauge
+                # which is why I wrapped the error like this
+                gitlab_token_expiration.remove(pat.name)
 
 
 def run(dry_run, wait_for_pipeline):
@@ -513,7 +577,11 @@ def run(dry_run, wait_for_pipeline):
     default_enable_closing = False
     instance = queries.get_gitlab_instance()
     settings = queries.get_app_interface_settings()
+    with GitLabApi(instance, settings=settings) as gl:
+        publish_access_token_expiration_metrics(gl)
     repos = queries.get_repos_gitlab_housekeeping(server=instance["url"])
+    repos = [r for r in repos if is_in_shard(r["url"])]
+    app_sre_usernames: Set[str] = set()
 
     for repo in repos:
         hk = repo["housekeeping"]
@@ -531,17 +599,40 @@ def run(dry_run, wait_for_pipeline):
             }
         )
         with GitLabApi(instance, project_url=project_url, settings=settings) as gl:
-            handle_stale_items(dry_run, gl, days_interval, enable_closing, "issue")
+            if not app_sre_usernames:
+                app_sre_usernames = get_app_sre_usernames(gl)
+            issues = gl.get_issues(state=MRState.OPENED)
             handle_stale_items(
-                dry_run, gl, days_interval, enable_closing, "merge-request"
+                dry_run,
+                gl,
+                days_interval,
+                enable_closing,
+                issues,
+                "issue",
             )
+            opened_merge_requests = gl.get_merge_requests(state=MRState.OPENED)
+            handle_stale_items(
+                dry_run,
+                gl,
+                days_interval,
+                enable_closing,
+                opened_merge_requests,
+                "merge-request",
+            )
+            project_merge_requests = [
+                mr for mr in opened_merge_requests if mr.state == MRState.OPENED
+            ]
+            reload_toggle = ReloadToggle(reload=False)
             rebase = hk.get("rebase")
             try:
                 merge_merge_requests(
                     dry_run,
                     gl,
+                    project_merge_requests,
+                    reload_toggle,
                     limit,
                     rebase,
+                    app_sre_usernames,
                     pipeline_timeout,
                     insist=True,
                     wait_for_pipeline=wait_for_pipeline,
@@ -556,8 +647,11 @@ def run(dry_run, wait_for_pipeline):
                 merge_merge_requests(
                     dry_run,
                     gl,
+                    project_merge_requests,
+                    reload_toggle,
                     limit,
                     rebase,
+                    app_sre_usernames,
                     pipeline_timeout,
                     wait_for_pipeline=wait_for_pipeline,
                     gl_instance=instance,

reconcile/gitlab_labeler.py

@@ -1,7 +1,9 @@
 import logging
 import os
-from collections.abc import Iterable
-from typing import Optional
+from collections.abc import (
+    Iterable,
+    Set,
+)
 
 from reconcile import queries
 from reconcile.gitlab_housekeeping import (
@@ -45,7 +47,7 @@ def get_parents_list() -> set[str]:
 
 def guess_onboarding_status(
     changed_paths: Iterable[str], apps: dict[str, dict], parent_apps: set[str]
-) -> Optional[str]:
+) -> str | None:
     """
     Guess the service name of a given MR from the changed paths of the
     MR. This will allow to add the onboarding status to the MR's as label
@@ -65,12 +67,11 @@ def guess_onboarding_status(
                 else:
                     app = apps[app_name]
                     labels.add(app["onboardingStatus"])
+            elif app_name in apps:
+                app = apps[app_name]
+                labels.add(app["onboardingStatus"])
             else:
-                if app_name in apps:
-                    app = apps[app_name]
-                    labels.add(app["onboardingStatus"])
-                else:
-                    logging.debug("Error getting app name " + path)
+                logging.debug("Error getting app name " + path)
 
     if len(labels) == 1:
         return labels.pop()
@@ -79,7 +80,7 @@ def guess_onboarding_status(
 
 def guess_labels(
     project_labels: Iterable[str], changed_paths: Iterable[str]
-) -> Iterable[str]:
+) -> Set[str]:
     """
     Guess labels returns a list of labels from the project labels
     that contain parts of the changed paths.
@@ -126,11 +127,11 @@ def run(dry_run, gitlab_project_id=None, gitlab_merge_request_id=None) -> None:
     settings = queries.get_app_interface_settings()
     with GitLabApi(instance, project_id=gitlab_project_id, settings=settings) as gl:
         project_labels = gl.get_project_labels()
-        labels = gl.get_merge_request_labels(gitlab_merge_request_id)
-        changed_paths = gl.get_merge_request_changed_paths(gitlab_merge_request_id)
+        merge_request = gl.get_merge_request(gitlab_merge_request_id)
+        changed_paths = gl.get_merge_request_changed_paths(merge_request)
         guessed_labels = guess_labels(project_labels, changed_paths)
-        labels_to_add = [b for b in guessed_labels if b not in labels]
-        labels_to_create = [b for b in labels_to_add if b not in project_labels]
+        labels_to_add = guessed_labels - set(merge_request.labels)
+        labels_to_create = labels_to_add - project_labels
 
         # This integration cannot check dry-run mode as it's always running with
         # dry_run flag to true.
@@ -141,4 +142,4 @@ def run(dry_run, gitlab_project_id=None, gitlab_merge_request_id=None) -> None:
 
         if labels_to_add:
             logging.info(["add_labels", labels_to_add])
-            gl.add_labels_to_merge_request(gitlab_merge_request_id, labels_to_add)
+            gl.add_labels_to_merge_request(merge_request, labels_to_add)