qontract-reconcile 0.10.2.dev33__tar.gz → 0.10.2.dev35__tar.gz

{qontract_reconcile-0.10.2.dev33 → qontract_reconcile-0.10.2.dev35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev33
+Version: 0.10.2.dev35
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev33 → qontract_reconcile-0.10.2.dev35}/reconcile/gitlab_housekeeping.py

@@ -49,6 +49,7 @@ from reconcile.utils.mr.labels import (
     prioritized_approval_label,
 )
 from reconcile.utils.sharding import is_in_shard
+from reconcile.utils.state import State, init_state
 
 MERGE_LABELS_PRIORITY = [
     prioritized_approval_label(p.value) for p in ChangeTypePriority
@@ -71,7 +72,6 @@ QONTRACT_INTEGRATION = "gitlab-housekeeping"
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 EXPIRATION_DATE_FORMAT = "%Y-%m-%d"
 
-
 merged_merge_requests = Counter(
     name="qontract_reconcile_merged_merge_requests",
     documentation="Number of merge requests that have been successfully merged in a repository",
@@ -156,16 +156,12 @@ def get_timed_out_pipelines(
 
 def clean_pipelines(
     dry_run: bool,
-    gl_instance: str,
-    gl_project_id: int,
-    gl_settings: str,
+    gl: GitLabApi,
+    fork_project_id: int,
     pipelines: list[dict],
 ) -> None:
     if not dry_run:
-        with GitLabApi(
-            gl_instance, project_id=gl_project_id, settings=gl_settings
-        ) as gl:
-            gl_piplelines = gl.project.pipelines
+        gl_piplelines = gl.get_project_by_id(fork_project_id).pipelines
 
     for p in pipelines:
         logging.info(["canceling", p["web_url"]])
@@ -179,6 +175,66 @@ def clean_pipelines(
                 )
 
 
+def verify_on_demand_tests(
+    dry_run: bool,
+    mr: ProjectMergeRequest,
+    must_pass: Iterable[str],
+    gl: GitLabApi,
+    state: State,
+) -> bool:
+    """
+    Check if MR has passed all necessary test jobs and add comments to indicate test results.
+    """
+    pipelines = gl.get_merge_request_pipelines(mr)
+    running_pipelines = [p for p in pipelines if p["status"] == "running"]
+    if running_pipelines:
+        # wait for pipelines complate
+        return False
+
+    commit = next(mr.commits())
+    fork_project = gl.get_project_by_id(mr.source_project_id)
+    statuses = fork_project.commits.get(commit.id).statuses.list()
+    test_state = {s.name: s.status for s in statuses}
+    remaining_tests = [t for t in must_pass if test_state.get(t) != "success"]
+    state_key = f"{gl.project.path_with_namespace}/{mr.iid}/{commit.id}"
+    # only add comment when state changes
+    state_change = state.get(state_key, None) != remaining_tests
+
+    if remaining_tests:
+        logging.info([
+            "on-demand tests",
+            "add comment",
+            gl.project.name,
+            mr.iid,
+            commit.id,
+        ])
+        if not dry_run and state_change:
+            markdown_report = (
+                f"On-demand Tests: \n\n For latest [commit]({commit.web_url}) You will need to pass following test jobs to get this MR merged.\n\n"
+                f"Add comment with `/test [test_name]` to trigger the tests.\n\n"
+            )
+            markdown_report += f"* {', '.join(remaining_tests)}\n"
+            gl.delete_merge_request_comments(mr, startswith="On-demand Tests:")
+            gl.add_comment_to_merge_request(mr, markdown_report)
+            state.add(state_key, remaining_tests, force=True)
+        return False
+    else:
+        # no remain_tests, pass the check
+        logging.info([
+            "on-demand tests",
+            "check pass",
+            gl.project.name,
+            mr.iid,
+            commit.id,
+        ])
+        if not dry_run and state_change:
+            markdown_report = f"On-demand Tests: \n\n All necessary tests have paased for latest [commit]({commit.web_url})\n"
+            gl.delete_merge_request_comments(mr, startswith="On-demand Tests:")
+            gl.add_comment_to_merge_request(mr, markdown_report)
+            state.add(state_key, remaining_tests, force=True)
+        return True
+
+
 def close_item(
     dry_run: bool,
     gl: GitLabApi,
@@ -291,6 +347,7 @@ def is_rebased(mr, gl: GitLabApi) -> bool:
 def get_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
+    state: State,
     users_allowed_to_label: Iterable[str] | None = None,
 ) -> list[dict[str, Any]]:
     mrs = gl.get_merge_requests(state=MRState.OPENED)
@@ -298,6 +355,7 @@ def get_merge_requests(
         dry_run=dry_run,
         gl=gl,
         project_merge_requests=mrs,
+        state=state,
         users_allowed_to_label=users_allowed_to_label,
     )
 
@@ -306,7 +364,9 @@ def preprocess_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
     project_merge_requests: list[ProjectMergeRequest],
+    state: State,
     users_allowed_to_label: Iterable[str] | None = None,
+    must_pass: Iterable[str] | None = None,
 ) -> list[dict[str, Any]]:
     results = []
     for mr in project_merge_requests:
@@ -320,6 +380,15 @@ def preprocess_merge_requests(
         if len(mr.commits()) == 0:
             continue
 
+        if must_pass and not verify_on_demand_tests(
+            dry_run=dry_run,
+            mr=mr,
+            must_pass=must_pass,
+            gl=gl,
+            state=state,
+        ):
+            continue
+
         labels = set(mr.labels)
         if not labels:
             continue
@@ -402,13 +471,18 @@ def rebase_merge_requests(
     rebase_limit,
     pipeline_timeout=None,
     wait_for_pipeline=False,
-    gl_instance=None,
-    gl_settings=None,
     users_allowed_to_label=None,
+    state=None,
 ):
     rebases = 0
     merge_requests = [
-        item["mr"] for item in get_merge_requests(dry_run, gl, users_allowed_to_label)
+        item["mr"]
+        for item in get_merge_requests(
+            dry_run=dry_run,
+            gl=gl,
+            state=state,
+            users_allowed_to_label=users_allowed_to_label,
+        )
     ]
     for mr in merge_requests:
         if is_rebased(mr, gl):
@@ -421,11 +495,10 @@ def rebase_merge_requests(
             timed_out_pipelines = get_timed_out_pipelines(pipelines, pipeline_timeout)
             if timed_out_pipelines:
                 clean_pipelines(
-                    dry_run,
-                    gl_instance,
-                    mr.source_project_id,
-                    gl_settings,
-                    timed_out_pipelines,
+                    dry_run=dry_run,
+                    gl=gl,
+                    fork_project_id=mr.source_project_id,
+                    pipelines=timed_out_pipelines,
                 )
 
         if wait_for_pipeline:
@@ -474,15 +547,20 @@ def merge_merge_requests(
     pipeline_timeout=None,
     insist=False,
     wait_for_pipeline=False,
-    gl_instance=None,
-    gl_settings=None,
     users_allowed_to_label=None,
+    must_pass=None,
+    state=None,
 ):
     merges = 0
     if reload_toggle.reload:
         project_merge_requests = gl.get_merge_requests(state=MRState.OPENED)
     merge_requests = preprocess_merge_requests(
-        dry_run, gl, project_merge_requests, users_allowed_to_label
+        dry_run=dry_run,
+        gl=gl,
+        project_merge_requests=project_merge_requests,
+        state=state,
+        users_allowed_to_label=users_allowed_to_label,
+        must_pass=must_pass,
     )
     merge_requests_waiting.labels(gl.project.id).set(len(merge_requests))
 
@@ -500,11 +578,10 @@ def merge_merge_requests(
             timed_out_pipelines = get_timed_out_pipelines(pipelines, pipeline_timeout)
             if timed_out_pipelines:
                 clean_pipelines(
-                    dry_run,
-                    gl_instance,
-                    mr.source_project_id,
-                    gl_settings,
-                    timed_out_pipelines,
+                    dry_run=dry_run,
+                    gl=gl,
+                    fork_project_id=mr.source_project_id,
+                    pipelines=timed_out_pipelines,
                 )
 
         if wait_for_pipeline:
@@ -582,6 +659,7 @@ def run(dry_run, wait_for_pipeline):
     repos = queries.get_repos_gitlab_housekeeping(server=instance["url"])
     repos = [r for r in repos if is_in_shard(r["url"])]
     app_sre_usernames: Set[str] = set()
+    state = init_state(QONTRACT_INTEGRATION)
 
     for repo in repos:
         hk = repo["housekeeping"]
@@ -624,6 +702,7 @@ def run(dry_run, wait_for_pipeline):
             ]
             reload_toggle = ReloadToggle(reload=False)
             rebase = hk.get("rebase")
+            must_pass = hk.get("must_pass")
             try:
                 merge_merge_requests(
                     dry_run,
@@ -636,9 +715,9 @@ def run(dry_run, wait_for_pipeline):
                     pipeline_timeout,
                     insist=True,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    must_pass=must_pass,
+                    state=state,
                 )
             except Exception:
                 logging.error(
@@ -654,9 +733,9 @@ def run(dry_run, wait_for_pipeline):
                     app_sre_usernames,
                     pipeline_timeout,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    must_pass=must_pass,
+                    state=state,
                 )
             if rebase:
                 rebase_merge_requests(
@@ -665,7 +744,6 @@ def run(dry_run, wait_for_pipeline):
                     limit,
                     pipeline_timeout=pipeline_timeout,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    state=state,
                 )

{qontract_reconcile-0.10.2.dev33 → qontract_reconcile-0.10.2.dev35}/reconcile/gql_definitions/introspection.json

@@ -20519,6 +20519,26 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "must_pass",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "SCALAR",
+                                        "name": "String",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,

{qontract_reconcile-0.10.2.dev33 → qontract_reconcile-0.10.2.dev35}/reconcile/queries.py

@@ -1648,6 +1648,7 @@ APPS_QUERY = """
             }
           }
         }
+        must_pass
       }
       jira {
         serverUrl

{qontract_reconcile-0.10.2.dev33 → qontract_reconcile-0.10.2.dev35}/reconcile/utils/gitlab_api.py

@@ -394,6 +394,10 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
             project = None
         return project
 
+    def get_project_by_id(self, project_id: int) -> Project:
+        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
+        return self.gl.projects.get(project_id)
+
     def get_issues(self, state):
         return self.get_items(self.project.issues.list, state=state)