qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/change_owners/change_owners.py

@@ -2,21 +2,25 @@ import logging
 import sys
 import traceback
 
+from gitlab.v4.objects import ProjectMergeRequest
+
 from reconcile import queries
 from reconcile.change_owners.approver import GqlApproverResolver
 from reconcile.change_owners.bundle import (
-    BundleFileType,
     FileDiffResolver,
     QontractServerFileDiffResolver,
 )
 from reconcile.change_owners.change_types import (
-    BundleFileChange,
+    ChangeTypeContext,
     ChangeTypePriority,
     ChangeTypeProcessor,
-    create_bundle_file_change,
-    get_priority_for_changes,
     init_change_type_processors,
 )
+from reconcile.change_owners.changes import (
+    BundleFileChange,
+    fetch_bundle_changes,
+    get_priority_for_changes,
+)
 from reconcile.change_owners.decision import (
     ChangeDecision,
     DecisionCommand,
@@ -28,18 +32,17 @@ from reconcile.change_owners.implicit_ownership import (
 )
 from reconcile.change_owners.self_service_roles import (
     cover_changes_with_self_service_roles,
+    fetch_self_service_roles,
 )
-from reconcile.gql_definitions.change_owners.queries import (
-    change_types,
-    self_service_roles,
-)
-from reconcile.gql_definitions.change_owners.queries.self_service_roles import RoleV1
+from reconcile.gql_definitions.change_owners.queries import change_types
 from reconcile.utils import gql
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.labels import (
     HOLD,
     NOT_SELF_SERVICEABLE,
+    RESTRICTED,
     SELF_SERVICEABLE,
+    change_owner_label,
     prioritized_approval_label,
 )
 from reconcile.utils.output import format_table
@@ -49,6 +52,10 @@ QONTRACT_INTEGRATION = "change-owners"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
 
 
+class NotAdmittedError(Exception):
+    pass
+
+
 def cover_changes(
     changes: list[BundleFileChange],
     change_type_processors: list[ChangeTypeProcessor],
@@ -77,33 +84,6 @@ def cover_changes(
     )
 
 
-def validate_self_service_role(role: RoleV1) -> None:
-    for ssc in role.self_service or []:
-        if ssc.change_type.context_schema:
-            # check that all referenced datafiles have a schema that
-            # is compatible with the change-type
-            incompatible_datafiles = [
-                df.path
-                for df in ssc.datafiles or []
-                if df.datafile_schema != ssc.change_type.context_schema
-            ]
-            if incompatible_datafiles:
-                raise ValueError(
-                    f"The datafiles {incompatible_datafiles} are not compatible with the "
-                    f"{ssc.change_type.name} change-types contextSchema {ssc.change_type.context_schema}"
-                )
-
-
-def fetch_self_service_roles(gql_api: gql.GqlApi) -> list[RoleV1]:
-    roles: list[RoleV1] = []
-    for r in self_service_roles.query(gql_api.query).roles or []:
-        if not r.self_service:
-            continue
-        validate_self_service_role(r)
-        roles.append(r)
-    return roles
-
-
 def fetch_change_type_processors(
     gql_api: gql.GqlApi, file_diff_resolver: FileDiffResolver
 ) -> list[ChangeTypeProcessor]:
@@ -113,60 +93,6 @@ def fetch_change_type_processors(
     )
 
 
-def fetch_bundle_changes(comparison_sha: str) -> list[BundleFileChange]:
-    """
-    reaches out to the qontract-server diff endpoint to find the files that
-    changed within two bundles (the current one representing the MR and the
-    explicitely passed comparision bundle - usually the state of the master branch).
-    """
-    changes = gql.get_diff(comparison_sha)
-    return _parse_bundle_changes(changes)
-
-
-def _parse_bundle_changes(bundle_changes) -> list[BundleFileChange]:
-    """
-    parses the output of the qontract-server /diff endpoint
-    """
-    datafiles = bundle_changes["datafiles"].values()
-    resourcefiles = bundle_changes["resources"].values()
-    logging.debug(
-        f"bundle contains {len(datafiles)} changed datafiles and {len(resourcefiles)} changed resourcefiles"
-    )
-
-    change_list = []
-    for c in datafiles:
-        bc = create_bundle_file_change(
-            path=c.get("datafilepath"),
-            schema=c.get("datafileschema"),
-            file_type=BundleFileType.DATAFILE,
-            old_file_content=c.get("old"),
-            new_file_content=c.get("new"),
-        )
-        if bc is not None:
-            change_list.append(bc)
-        else:
-            logging.debug(
-                f"skipping datafile {c.get('datafilepath')} - no changes detected"
-            )
-
-    for c in resourcefiles:
-        bc = create_bundle_file_change(
-            path=c.get("resourcepath"),
-            schema=c.get("new", {}).get("$schema", c.get("old", {}).get("$schema")),
-            file_type=BundleFileType.RESOURCEFILE,
-            old_file_content=c.get("old", {}).get("content"),
-            new_file_content=c.get("new", {}).get("content"),
-        )
-        if bc is not None:
-            change_list.append(bc)
-        else:
-            logging.debug(
-                f"skipping resourcefile {c.get('resourcepath')} - no changes detected"
-            )
-
-    return change_list
-
-
 CHANGE_TYPE_PROCESSING_MODE_LIMITED = "limited"
 CHANGE_TYPE_PROCESSING_MODE_AUTHORITATIVE = "authoritative"
 
@@ -175,7 +101,7 @@ def manage_conditional_label(
     current_labels: list[str],
     conditional_labels: dict[str, bool],
     dry_run: bool = True,
-) -> list[str]:
+) -> set[str]:
     new_labels = current_labels.copy()
     for label, condition in conditional_labels.items():
         if condition and label not in new_labels:
@@ -186,14 +112,15 @@ def manage_conditional_label(
             logging.info(f"removing label {label}")
             if not dry_run:
                 new_labels.remove(label)
-    return new_labels
+    return set(new_labels)
 
 
 def write_coverage_report_to_mr(
     self_serviceable: bool,
     change_decisions: list[ChangeDecision],
+    change_admitted: bool,
     authoritative: bool,
-    mr_id: int,
+    merge_request: ProjectMergeRequest,
     gl: GitLabApi,
 ) -> None:
     """
@@ -202,27 +129,28 @@ def write_coverage_report_to_mr(
     a new one.
     """
     change_coverage_report_header = "Change coverage report"
-    comments = gl.get_merge_request_comments(mr_id)
     # delete previous report comment
-    for c in sorted(comments, key=lambda k: k["created_at"]):
-        body = c["body"] or ""
-        if c["username"] == gl.user.username and body.startswith(
-            change_coverage_report_header
-        ):
-            gl.delete_gitlab_comment(mr_id, c["id"])
+    gl.delete_merge_request_comments(
+        merge_request,
+        startswith=change_coverage_report_header,
+    )
 
     # add new report comment
     results = []
     approver_reachability = set()
     for d in change_decisions:
         approvers = [
-            f"{ctctx.context} - { ' '.join([f'@{a.org_username}' if a.tag_on_merge_requests else a.org_username for a in ctctx.approvers]) }"
-            for ctctx in d.deduped_coverage()
+            f"{cr.context} - {' '.join([f'@{a.org_username}' if a.tag_on_merge_requests else a.org_username for a in cr.approvers])}"
+            for cr in d.change_responsibles
         ]
-        for cctx in d.deduped_coverage():
-            approver_reachability.update(
-                {ar.render_for_mr_report() for ar in cctx.approver_reachability or []}
+        if d.coverable_by_fragment_decisions:
+            approvers.append(
+                "automatically approved if all sub-properties are approved"
             )
+        for cr in d.change_responsibles:
+            approver_reachability.update({
+                ar.render_for_mr_report() for ar in cr.approver_reachability or []
+            })
         if not approvers:
             approvers = ["[- not self-serviceable -]"]
         item = {
@@ -230,7 +158,9 @@ def write_coverage_report_to_mr(
             "schema": d.file.schema,
             "change": d.diff.path,
         }
-        if d.is_held():
+        if not change_admitted:
+            item["status"] = "restricted"
+        elif d.is_held():
             item["status"] = "hold"
         elif d.is_approved():
             item["status"] = "approved"
@@ -242,16 +172,26 @@ def write_coverage_report_to_mr(
 
     self_serviceability_hint = "All changes require an `/lgtm` from a listed approver "
     if not self_serviceable:
-        self_serviceability_hint += "but <b>not all changes are self-serviceable and require AppSRE approval</b>"
+        self_serviceability_hint += (
+            "but <b>not all changes are self-serviceable and require AppSRE approval</b>."
+            "The AppSRE Interrupt Catcher (IC) will review your Merge Request (MR) as it comes up in their "
+            "<a href='https://gitlab.cee.redhat.com/service/app-interface-output/-/blob/master/app-interface-review-queue.md'>queue</a>, "
+            "please do not ping them directly unless this is <b>urgent</b>."
+            "\nPlease see https://gitlab.cee.redhat.com/service/app-interface#app-interface-etiquette for more information. Thank you :)"
+        )
     if not authoritative:
         self_serviceability_hint += "\n\nchanges outside of data and resources detected - <b>PAY EXTRA ATTENTION WHILE REVIEWING</b>\n\n"
+
+    if not change_admitted:
+        self_serviceability_hint += "\n\nchanges are not admitted. Please request `/good-to-test` from one of the approvers.\n\n"
+
     approver_reachability_hint = "Reach out to approvers for reviews"
     if approver_reachability:
-        approver_reachability_hint += " on\n" + "\n".join(
-            [f"* {ar}" for ar in approver_reachability or []]
-        )
+        approver_reachability_hint += " on\n" + "\n".join([
+            f"* {ar}" for ar in approver_reachability or []
+        ])
     gl.add_comment_to_merge_request(
-        mr_id,
+        merge_request,
         f"{change_coverage_report_header}<br/>"
         f"{self_serviceability_hint}\n"
         f"{coverage_report}\n\n"
@@ -263,33 +203,38 @@ def write_coverage_report_to_mr(
 def write_coverage_report_to_stdout(change_decisions: list[ChangeDecision]) -> None:
     results = []
     for d in change_decisions:
+        if d.coverable_by_fragment_decisions:
+            results.append({
+                "file": d.file.path,
+                "schema": d.file.schema,
+                "changed path": d.diff.path,
+                "change type": "...",
+                "origin": "...",
+                "context": "coverable by fragments",
+                "approver_reachability": "",
+                "disabled": False,
+            })
         if d.coverage:
             for ctx in d.coverage:
-                results.append(
-                    {
-                        "file": d.file.path,
-                        "schema": d.file.schema,
-                        "changed path": d.diff.path,
-                        "change type": ctx.change_type_processor.name,
-                        "origin": ctx.origin,
-                        "context": ctx.context,
-                        "approver_reachability": ", ".join(
-                            [
-                                ar.render_for_mr_report()
-                                for ar in ctx.approver_reachability or []
-                            ]
-                        ),
-                        "disabled": str(ctx.disabled),
-                    }
-                )
-        else:
-            results.append(
-                {
+                results.append({
                     "file": d.file.path,
                     "schema": d.file.schema,
                     "changed path": d.diff.path,
-                }
-            )
+                    "change type": ctx.change_type_processor.name,
+                    "origin": ctx.origin,
+                    "context": ctx.context,
+                    "approver_reachability": ", ".join([
+                        ar.render_for_mr_report()
+                        for ar in ctx.approver_reachability or []
+                    ]),
+                    "disabled": str(ctx.disabled),
+                })
+        else:
+            results.append({
+                "file": d.file.path,
+                "schema": d.file.schema,
+                "changed path": d.diff.path,
+            })
 
     print(
         format_table(
@@ -313,6 +258,41 @@ def init_gitlab(gitlab_project_id: str) -> GitLabApi:
     return GitLabApi(instance, project_id=gitlab_project_id, settings=settings)
 
 
+def is_coverage_admitted(
+    coverage: ChangeTypeContext, mr_author: str, good_to_test_approvers: set[str]
+) -> bool:
+    return any(
+        a.org_username == mr_author or a.org_username in good_to_test_approvers
+        for a in coverage.approvers
+    )
+
+
+def is_change_admitted(
+    changes: list[BundleFileChange], mr_author: str, good_to_test_approvers: set[str]
+) -> bool:
+    # Checks if mr authors are allowed to do the changes in the merge request.
+    # If a change type is restrictive and the author is not an approver,
+    # this is not admitted.
+    # A change might be admitted if a user that has the restrictive change
+    # type is an approver or an approver adds an /good-to-test comment.
+
+    restrictive_coverages = [
+        c
+        for change in changes
+        for dc in change.diff_coverage
+        for c in dc.coverage
+        if c.change_type_processor.restrictive
+    ]
+
+    change_types_to_approve = {c.origin for c in restrictive_coverages}
+    change_types_approved = {
+        c.origin
+        for c in restrictive_coverages
+        if is_coverage_admitted(c, mr_author, good_to_test_approvers)
+    }
+    return change_types_to_approve == change_types_approved
+
+
 def run(
     dry_run: bool,
     gitlab_project_id: str,
@@ -376,7 +356,9 @@ def run(
         #
         changes = fetch_bundle_changes(comparison_sha)
         logging.info(
-            f"detected {len(changes)} changed files with {sum(c.raw_diff_count() for c in changes)} differences"
+            f"detected {len(changes)} changed files "
+            f"with {sum(c.raw_diff_count() for c in changes)} differences "
+            f"and {len([c for c in changes if c.metadata_only_change])} metadata-only changes"
         )
         cover_changes(
             changes,
@@ -393,75 +375,103 @@ def run(
         #   D E C I S I O N S
         #
 
-        gl = init_gitlab(gitlab_project_id)
-        approver_decisions = get_approver_decisions_from_mr_comments(
-            gl.get_merge_request_comments(
-                gitlab_merge_request_id, include_description=True
-            )
-        )
-        change_decisions = apply_decisions_to_changes(
-            changes,
-            approver_decisions,
-            {
-                gl.user.username,
-                gl.get_merge_request_author_username(gitlab_merge_request_id),
-            },
-        )
-        hold = any(d.is_held() for d in change_decisions)
-        approved = all(d.is_approved() and not d.is_held() for d in change_decisions)
+        with init_gitlab(gitlab_project_id) as gl:
+            merge_request = gl.get_merge_request(gitlab_merge_request_id)
 
-        #
-        #   R E P O R T I N G
-        #
+            comments = gl.get_merge_request_comments(merge_request)
+            good_to_test_approvers = {
+                c["username"] for c in comments if c["body"].strip() == "/good-to-test"
+            }
 
-        if mr_management_enabled:
-            write_coverage_report_to_mr(
-                self_serviceable,
-                change_decisions,
-                change_type_processing_mode
-                == CHANGE_TYPE_PROCESSING_MODE_AUTHORITATIVE,
-                gitlab_merge_request_id,
-                gl,
+            change_admitted = is_change_admitted(
+                changes,
+                gl.get_merge_request_author_username(merge_request),
+                good_to_test_approvers,
+            )
+            approver_decisions = get_approver_decisions_from_mr_comments(
+                gl.get_merge_request_comments(merge_request, include_description=True)
+            )
+            change_decisions = apply_decisions_to_changes(
+                changes,
+                approver_decisions,
+                {
+                    gl.user.username,
+                    gl.get_merge_request_author_username(merge_request),
+                },
+            )
+            hold = any(d.is_held() for d in change_decisions)
+            approved = all(
+                d.is_approved() and not d.is_held() for d in change_decisions
             )
-        write_coverage_report_to_stdout(change_decisions)
-
-        #
-        #   L A B E L I N G
-        #
-
-        labels = gl.get_merge_request_labels(gitlab_merge_request_id)
 
-        # base labels
-        conditional_labels = {
-            SELF_SERVICEABLE: self_serviceable,
-            NOT_SELF_SERVICEABLE: not self_serviceable,
-            HOLD: self_serviceable and hold,
-        }
+            #
+            #   R E P O R T I N G
+            #
+
+            if mr_management_enabled:
+                write_coverage_report_to_mr(
+                    self_serviceable,
+                    change_decisions,
+                    change_admitted,
+                    change_type_processing_mode
+                    == CHANGE_TYPE_PROCESSING_MODE_AUTHORITATIVE,
+                    merge_request,
+                    gl,
+                )
+            write_coverage_report_to_stdout(change_decisions)
+
+            #
+            #   L A B E L I N G
+            #
+
+            # base labels
+            conditional_labels = {
+                SELF_SERVICEABLE: self_serviceable,
+                NOT_SELF_SERVICEABLE: not self_serviceable,
+                HOLD: self_serviceable and hold,
+                RESTRICTED: not change_admitted,
+            }
 
-        # priority labels
-        mr_priority = get_priority_for_changes(changes)
-        conditional_labels.update(
-            {
+            # priority labels
+            mr_priority = get_priority_for_changes(changes)
+            conditional_labels.update({
                 prioritized_approval_label(p.value): self_serviceable
                 and approved
                 and p == mr_priority
                 for p in ChangeTypePriority
-            }
-        )
-        labels = manage_conditional_label(
-            current_labels=labels, conditional_labels=conditional_labels, dry_run=False
-        )
-        if mr_management_enabled:
-            gl.set_labels_on_merge_request(gitlab_merge_request_id, labels)
-        else:
-            # if MR management is disabled, we need to make sure the self-serviceable
-            # labels is not present, because other integration react to them
-            # e.g. gitlab-housekeeper rejects direct lgtm labels and the review-queue
-            # skips MRs with this label
-            if SELF_SERVICEABLE in labels:
-                gl.remove_label_from_merge_request(
-                    gitlab_merge_request_id, SELF_SERVICEABLE
-                )
+            })
+            labels = manage_conditional_label(
+                current_labels=merge_request.labels,
+                conditional_labels=conditional_labels,
+                dry_run=False,
+            )
 
+            # change-owner labels
+            labels = {
+                co_label
+                for co_label in labels
+                if not co_label.startswith("change-owner/")
+            }
+            for bc in changes:
+                for label in bc.change_owner_labels:
+                    labels.add(change_owner_label(label))
+
+            if mr_management_enabled:
+                gl.set_labels_on_merge_request(merge_request, labels)
+            elif SELF_SERVICEABLE in labels:
+                # if MR management is disabled, we need to make sure the self-serviceable
+                # labels is not present, because other integration react to them
+                # e.g. gitlab-housekeeper rejects direct lgtm labels and the review-queue
+                # skips MRs with this label
+                gl.remove_label(merge_request, SELF_SERVICEABLE)
+
+            if not change_admitted:
+                raise NotAdmittedError("Change not admitted")
+
+    except NotAdmittedError as e:
+        # This is not an error, but we want to fail the integration, since the
+        # MR author is not allowed to do this change
+        logging.error(e)
+        sys.exit(1)
     except BaseException:
         logging.error(traceback.format_exc())