qontract-reconcile 0.10.2.dev32__py3-none-any.whl → 0.10.2.dev34__py3-none-any.whl

{qontract_reconcile-0.10.2.dev32.dist-info → qontract_reconcile-0.10.2.dev34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev32
+Version: 0.10.2.dev34
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev32.dist-info → qontract_reconcile-0.10.2.dev34.dist-info}/RECORD

@@ -30,7 +30,7 @@ reconcile/github_repo_permissions_validator.py,sha256=dcbXdUx6imjNchjp3pg9-z1i7l
 reconcile/github_users.py,sha256=nfTq78QRONIfDVj-5O3bD6psllJjzWFnog-EJ1WqFPU,3672
 reconcile/github_validator.py,sha256=cVTVxJIGR4a1Jz8wrdXEAb_CMpXUzvykVmUURX4cook,917
 reconcile/gitlab_fork_compliance.py,sha256=c7UfqSAsW04c1bWJmXXaQDwtUcG4Kb6nCJAyRU2uAuw,4449
-reconcile/gitlab_housekeeping.py,sha256=NRiMo4l1K3JeP4A-qEF_VCGhadXlLuf5X_7qcTTtnbI,22482
+reconcile/gitlab_housekeeping.py,sha256=i4hNZViV2yfUTDNsYhVSLik211q8_c4nbTK8ilmpufc,25313
 reconcile/gitlab_labeler.py,sha256=4xJHmVX155fclrHqkR926sL1GH6RTN5XfZ8PnqNXbRA,4534
 reconcile/gitlab_members.py,sha256=MUIgYDLeJx2-_vMypyq2Pa17cpKdXATYhtVACS2ghpQ,8297
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
@@ -94,7 +94,7 @@ reconcile/quay_mirror.py,sha256=0KtQFwrvMNtlsPJ9F_-ICaVIjgIUjFxqipvAPcvyg3Q,1533
 reconcile/quay_mirror_org.py,sha256=tXKuF6JtmaNRwu8_g_65U_Vpd6sFBYeXmJA-flVhylE,10764
 reconcile/quay_permissions.py,sha256=9KOutS1w4RFQqkvMSy54VtsKNx56-phzP6yI_rEW-B8,4244
 reconcile/quay_repos.py,sha256=cuEYG0HUe0ut5yvLdEwOF5-CmccpXQHRb_wDazvDrvQ,6895
-reconcile/queries.py,sha256=of54G-Vqy-_deKdZcWs-B9LwxIdhQFUlQ198pBGCp3g,51488
+reconcile/queries.py,sha256=3nqVO5zv_TsnCK4jsJfF8N4Z7ADfgtUaTD2nv8EzlMs,51506
 reconcile/query_validator.py,sha256=MSh5pKLBksws4AqfuvT8nrIGucIbqX-IOzYyPYTLO7k,1491
 reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
 reconcile/resource_scraper.py,sha256=znXCHrU7YwPfKuxGBiUrV7T1tYtn4vlz9qmZlfy6Flg,2307
@@ -215,7 +215,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=GoGW7nosXPWvZYQsq6m8nYVIJHIO5hBh9upeFhS_M_k,2216702
+reconcile/gql_definitions/introspection.json,sha256=WWQ1wCVxaP7aNIvIttoxVfFtZkODTopzAUv_bCZVNR4,2217602
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -584,7 +584,7 @@ reconcile/utils/external_resources.py,sha256=YzTb0xAcNdmKO326mGQy7BmST56CZcdru4l
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=wzVIYAeKlMGW538U1mkJWUI6h_mFRUY4lawh2AR8hw4,2345
 reconcile/utils/github_api.py,sha256=y3fxty7FKvfhdzfHgGSaIstL6A_Y2loUcMiyIK5TMDg,2750
-reconcile/utils/gitlab_api.py,sha256=SR0graae7UBZyLLKA7yGsOqKm7dXFe5cexcaa1SokQo,28486
+reconcile/utils/gitlab_api.py,sha256=N1QdgF6Jdhg_nF03vtfouvqXP7MVfkPRnq7Q3z169kM,28662
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
@@ -727,7 +727,7 @@ reconcile/utils/runtime/sharding.py,sha256=r0ieUtNed7NvknSw6qQrCkKpVXE1shuHGnfFc
 reconcile/utils/saasherder/__init__.py,sha256=3U8plqMAPRE1kjwZ5YnIsYsggTf4_gS7flRUEuXVBAs,343
 reconcile/utils/saasherder/interfaces.py,sha256=C2wrw34OXypshVocAsPrVZsSHptgw4g9u7Haa2wulZQ,9087
 reconcile/utils/saasherder/models.py,sha256=z8ln03zi2a8cu716NcNUDHp8Dv1VcVbhqdWVxCl7x9A,10148
-reconcile/utils/saasherder/saasherder.py,sha256=6d3ZK_hNfv6hVxULh3BhXzTRMXoTR2YvjcoQQ1oKF8M,86099
+reconcile/utils/saasherder/saasherder.py,sha256=-Q1cMXuclv-1F1cMoy7ntzL_1_0C-X0q9sD6otfKbyw,86222
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=gRL1rQ0AqvShei_rcGqC3HDYGskOFKE1nPrJyJE9yno,4676
@@ -773,7 +773,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev32.dist-info/METADATA,sha256=Aj39BfBFhSBY0PanTtlnlcY2FpVeG_k_VWiCOo0Enbc,24665
-qontract_reconcile-0.10.2.dev32.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev32.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev32.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev34.dist-info/METADATA,sha256=gEINV-7WHJvNpdhF_iG3Mur5JD5aVPWJ_q2CghjgcQ4,24665
+qontract_reconcile-0.10.2.dev34.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev34.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev34.dist-info/RECORD,,

reconcile/gitlab_housekeeping.py

@@ -49,6 +49,7 @@ from reconcile.utils.mr.labels import (
     prioritized_approval_label,
 )
 from reconcile.utils.sharding import is_in_shard
+from reconcile.utils.state import State, init_state
 
 MERGE_LABELS_PRIORITY = [
     prioritized_approval_label(p.value) for p in ChangeTypePriority
@@ -71,7 +72,6 @@ QONTRACT_INTEGRATION = "gitlab-housekeeping"
 DATE_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
 EXPIRATION_DATE_FORMAT = "%Y-%m-%d"
 
-
 merged_merge_requests = Counter(
     name="qontract_reconcile_merged_merge_requests",
     documentation="Number of merge requests that have been successfully merged in a repository",
@@ -156,16 +156,12 @@ def get_timed_out_pipelines(
 
 def clean_pipelines(
     dry_run: bool,
-    gl_instance: str,
-    gl_project_id: int,
-    gl_settings: str,
+    gl: GitLabApi,
+    fork_project_id: int,
     pipelines: list[dict],
 ) -> None:
     if not dry_run:
-        with GitLabApi(
-            gl_instance, project_id=gl_project_id, settings=gl_settings
-        ) as gl:
-            gl_piplelines = gl.project.pipelines
+        gl_piplelines = gl.get_project_by_id(fork_project_id).pipelines
 
     for p in pipelines:
         logging.info(["canceling", p["web_url"]])
@@ -179,6 +175,66 @@ def clean_pipelines(
                 )
 
 
+def verify_on_demand_tests(
+    dry_run: bool,
+    mr: ProjectMergeRequest,
+    must_pass: Iterable[str],
+    gl: GitLabApi,
+    state: State,
+) -> bool:
+    """
+    Check if MR has passed all necessary test jobs and add comments to indicate test results.
+    """
+    pipelines = gl.get_merge_request_pipelines(mr)
+    running_pipelines = [p for p in pipelines if p["status"] == "running"]
+    if running_pipelines:
+        # wait for pipelines complate
+        return False
+
+    commit = next(mr.commits())
+    fork_project = gl.get_project_by_id(mr.source_project_id)
+    statuses = fork_project.commits.get(commit.id).statuses.list()
+    test_state = {s.name: s.status for s in statuses}
+    remaining_tests = [t for t in must_pass if test_state.get(t) != "success"]
+    state_key = f"{gl.project.path_with_namespace}/{mr.iid}/{commit.id}"
+    # only add comment when state changes
+    state_change = state.get(state_key, None) != remaining_tests
+
+    if remaining_tests:
+        logging.info([
+            "on-demand tests",
+            "add comment",
+            gl.project.name,
+            mr.iid,
+            commit.id,
+        ])
+        if not dry_run and state_change:
+            markdown_report = (
+                f"On-demand Tests: \n\n For latest [commit]({commit.web_url}) You will need to pass following test jobs to get this MR merged.\n\n"
+                f"Add comment with `/test [test_name]` to trigger the tests.\n\n"
+            )
+            markdown_report += f"* {', '.join(remaining_tests)}\n"
+            gl.delete_merge_request_comments(mr, startswith="On-demand Tests:")
+            gl.add_comment_to_merge_request(mr, markdown_report)
+            state.add(state_key, remaining_tests, force=True)
+        return False
+    else:
+        # no remain_tests, pass the check
+        logging.info([
+            "on-demand tests",
+            "check pass",
+            gl.project.name,
+            mr.iid,
+            commit.id,
+        ])
+        if not dry_run and state_change:
+            markdown_report = f"On-demand Tests: \n\n All necessary tests have paased for latest [commit]({commit.web_url})\n"
+            gl.delete_merge_request_comments(mr, startswith="On-demand Tests:")
+            gl.add_comment_to_merge_request(mr, markdown_report)
+            state.add(state_key, remaining_tests, force=True)
+        return True
+
+
 def close_item(
     dry_run: bool,
     gl: GitLabApi,
@@ -291,6 +347,7 @@ def is_rebased(mr, gl: GitLabApi) -> bool:
 def get_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
+    state: State,
     users_allowed_to_label: Iterable[str] | None = None,
 ) -> list[dict[str, Any]]:
     mrs = gl.get_merge_requests(state=MRState.OPENED)
@@ -298,6 +355,7 @@ def get_merge_requests(
         dry_run=dry_run,
         gl=gl,
         project_merge_requests=mrs,
+        state=state,
         users_allowed_to_label=users_allowed_to_label,
     )
 
@@ -306,7 +364,9 @@ def preprocess_merge_requests(
     dry_run: bool,
     gl: GitLabApi,
     project_merge_requests: list[ProjectMergeRequest],
+    state: State,
     users_allowed_to_label: Iterable[str] | None = None,
+    must_pass: Iterable[str] | None = None,
 ) -> list[dict[str, Any]]:
     results = []
     for mr in project_merge_requests:
@@ -320,6 +380,15 @@ def preprocess_merge_requests(
         if len(mr.commits()) == 0:
             continue
 
+        if must_pass and not verify_on_demand_tests(
+            dry_run=dry_run,
+            mr=mr,
+            must_pass=must_pass,
+            gl=gl,
+            state=state,
+        ):
+            continue
+
         labels = set(mr.labels)
         if not labels:
             continue
@@ -402,13 +471,18 @@ def rebase_merge_requests(
     rebase_limit,
     pipeline_timeout=None,
     wait_for_pipeline=False,
-    gl_instance=None,
-    gl_settings=None,
     users_allowed_to_label=None,
+    state=None,
 ):
     rebases = 0
     merge_requests = [
-        item["mr"] for item in get_merge_requests(dry_run, gl, users_allowed_to_label)
+        item["mr"]
+        for item in get_merge_requests(
+            dry_run=dry_run,
+            gl=gl,
+            state=state,
+            users_allowed_to_label=users_allowed_to_label,
+        )
     ]
     for mr in merge_requests:
         if is_rebased(mr, gl):
@@ -421,11 +495,10 @@ def rebase_merge_requests(
             timed_out_pipelines = get_timed_out_pipelines(pipelines, pipeline_timeout)
             if timed_out_pipelines:
                 clean_pipelines(
-                    dry_run,
-                    gl_instance,
-                    mr.source_project_id,
-                    gl_settings,
-                    timed_out_pipelines,
+                    dry_run=dry_run,
+                    gl=gl,
+                    fork_project_id=mr.source_project_id,
+                    pipelines=timed_out_pipelines,
                 )
 
         if wait_for_pipeline:
@@ -474,15 +547,20 @@ def merge_merge_requests(
     pipeline_timeout=None,
     insist=False,
     wait_for_pipeline=False,
-    gl_instance=None,
-    gl_settings=None,
     users_allowed_to_label=None,
+    must_pass=None,
+    state=None,
 ):
     merges = 0
     if reload_toggle.reload:
         project_merge_requests = gl.get_merge_requests(state=MRState.OPENED)
     merge_requests = preprocess_merge_requests(
-        dry_run, gl, project_merge_requests, users_allowed_to_label
+        dry_run=dry_run,
+        gl=gl,
+        project_merge_requests=project_merge_requests,
+        state=state,
+        users_allowed_to_label=users_allowed_to_label,
+        must_pass=must_pass,
     )
     merge_requests_waiting.labels(gl.project.id).set(len(merge_requests))
 
@@ -500,11 +578,10 @@ def merge_merge_requests(
             timed_out_pipelines = get_timed_out_pipelines(pipelines, pipeline_timeout)
             if timed_out_pipelines:
                 clean_pipelines(
-                    dry_run,
-                    gl_instance,
-                    mr.source_project_id,
-                    gl_settings,
-                    timed_out_pipelines,
+                    dry_run=dry_run,
+                    gl=gl,
+                    fork_project_id=mr.source_project_id,
+                    pipelines=timed_out_pipelines,
                 )
 
         if wait_for_pipeline:
@@ -582,6 +659,7 @@ def run(dry_run, wait_for_pipeline):
     repos = queries.get_repos_gitlab_housekeeping(server=instance["url"])
     repos = [r for r in repos if is_in_shard(r["url"])]
     app_sre_usernames: Set[str] = set()
+    state = init_state(QONTRACT_INTEGRATION)
 
     for repo in repos:
         hk = repo["housekeeping"]
@@ -624,6 +702,7 @@ def run(dry_run, wait_for_pipeline):
             ]
             reload_toggle = ReloadToggle(reload=False)
             rebase = hk.get("rebase")
+            must_pass = hk.get("must_pass")
             try:
                 merge_merge_requests(
                     dry_run,
@@ -636,9 +715,9 @@ def run(dry_run, wait_for_pipeline):
                     pipeline_timeout,
                     insist=True,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    must_pass=must_pass,
+                    state=state,
                 )
             except Exception:
                 logging.error(
@@ -654,9 +733,9 @@ def run(dry_run, wait_for_pipeline):
                     app_sre_usernames,
                     pipeline_timeout,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    must_pass=must_pass,
+                    state=state,
                 )
             if rebase:
                 rebase_merge_requests(
@@ -665,7 +744,6 @@ def run(dry_run, wait_for_pipeline):
                     limit,
                     pipeline_timeout=pipeline_timeout,
                     wait_for_pipeline=wait_for_pipeline,
-                    gl_instance=instance,
-                    gl_settings=settings,
                     users_allowed_to_label=users_allowed_to_label,
+                    state=state,
                 )

reconcile/gql_definitions/introspection.json

@@ -20519,6 +20519,26 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "must_pass",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "SCALAR",
+                                        "name": "String",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,

reconcile/queries.py

@@ -1648,6 +1648,7 @@ APPS_QUERY = """
             }
           }
         }
+        must_pass
       }
       jira {
         serverUrl

reconcile/utils/gitlab_api.py

@@ -394,6 +394,10 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
             project = None
         return project
 
+    def get_project_by_id(self, project_id: int) -> Project:
+        gitlab_request.labels(integration=INTEGRATION_NAME).inc()
+        return self.gl.projects.get(project_id)
+
     def get_issues(self, state):
         return self.get_items(self.project.issues.list, state=state)
 

reconcile/utils/saasherder/saasherder.py

@@ -1361,8 +1361,9 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
             except ResourceKeyExistsError:
                 ri.register_error()
                 msg = (
-                    f"[{spec.cluster}/{spec.namespace}] desired item "
-                    + f"already exists: {resource['kind']}/{resource['metadata']['name']}. "
+                    f"[{spec.cluster}/{spec.namespace}] Duplicate resources in your deployment template detected. "
+                    + "The following is defined multiple times in your deployment template: "
+                    + f"{resource['kind']}/{resource['metadata']['name']}. "
                     + f"saas file name: {spec.saas_file_name}, "
                     + "resource template name: "
                     + f"{spec.resource_template_name}."