qontract-reconcile 0.10.1rc596__py3-none-any.whl → 0.10.1rc597__py3-none-any.whl

{qontract_reconcile-0.10.1rc596.dist-info → qontract_reconcile-0.10.1rc597.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc596
+Version: 0.10.1rc597
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc596.dist-info → qontract_reconcile-0.10.1rc597.dist-info}/RECORD

@@ -9,7 +9,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=3THrA_8GdYIWiB6lwq8WZ8udxS6WALPB61xZZ40-puQ,89871
+reconcile/cli.py,sha256=eZfqiWuI1r_BEx_VzxTPYrqqs1PHiQPAnEt00qZ3vNo,91518
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -120,7 +120,7 @@ reconcile/vpc_peerings_validator.py,sha256=Kv22HJVlTW9l9GB2eXwjPWqdDbr_VuvQBNPtt
 reconcile/aus/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aus/advanced_upgrade_service.py,sha256=PkVcXBMrveW5euvqjEBO4e5-9KDb_6hszLI2GrWpx2w,21378
 reconcile/aus/aus_label_source.py,sha256=qoP8Fgxuu1tCuhG6ixCWve7Ll-KD6a79E2uLAmC0ifw,4184
-reconcile/aus/base.py,sha256=Qdktz7W7J9TWVCYJZHwyKoIf4sVI7W9o_WwzsOBxLfQ,44912
+reconcile/aus/base.py,sha256=ESycSlsxNGjpy1v2G2NXB7WB6NMYrXCaN4kSVMicRKk,44462
 reconcile/aus/cluster_version_data.py,sha256=j4UyEBi5mQuvPq5Lo7a_L_0blxvH790wJV07uAiikFU,7126
 reconcile/aus/metrics.py,sha256=fIew-rzi_kYuI5Gxn3-4bQVIr2oNibiKPyGnhB-xKU4,3538
 reconcile/aus/models.py,sha256=muBmbovxYtSNLFrTLVRcJYZ4dx6JLh8n3Q1-DjWJOHM,7098
@@ -128,6 +128,11 @@ reconcile/aus/ocm_addons_upgrade_scheduler_org.py,sha256=fshslI27hrqT40qrVsVOQaW
 reconcile/aus/ocm_upgrade_scheduler.py,sha256=7cK2SakCFkl5EdnqUEAYdUo4pUnnf-SsUR10uytAGyE,3058
 reconcile/aus/ocm_upgrade_scheduler_org.py,sha256=OBgE5mnVdQQV4tMH0AE2V_PDt9Gy6d-LyuPceqjORts,2331
 reconcile/aus/upgrades.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/aus/version_gate_approver.py,sha256=VJ6Lrzkapr14QvulzIsE-sTfawIDcyn9UGOC3pIM1gs,6895
+reconcile/aus/version_gates/__init__.py,sha256=fWx-IvS132Wpa4gWNIuoNvFwqhkuUuFWYWq5-xiLklI,362
+reconcile/aus/version_gates/handler.py,sha256=S_isQPYHbG4DERiUEvQBZ6ngiFX3uMmATA-Q_eNKmFk,839
+reconcile/aus/version_gates/ocp_gate_handler.py,sha256=RW1ppDaCZXVegV9AzzqYXxDUu_Z_7d43Z5h2Pk_piKc,716
+reconcile/aus/version_gates/sts_version_gate_handler.py,sha256=PhJ7yBh2q-rv9CJcfFhc0H11nyDyG7NAryNS3F74xdY,3697
 reconcile/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_ami_cleanup/integration.py,sha256=IW95cpMj2P5ffs-AxsR_TDQCJnYFBhLIfP2de7dz_8A,10109
 reconcile/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -535,7 +540,7 @@ reconcile/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/aggregated_list.py,sha256=pkYoBj7WwmaNgEefETqEOFTnQMcUzHE3mdsVdzGYj60,3372
 reconcile/utils/amtool.py,sha256=JV5-to_e_FaIcvJWTKYA9d6L3LwzwijM0MjUWn83eD4,2204
 reconcile/utils/aws_api.py,sha256=Wy040GBQ3HrWmtxe1QAx3zl1I3phVDVjXEx_OITEcOw,69191
-reconcile/utils/aws_helper.py,sha256=E8NHkStoHRmvLVjRll2f5kGtU3i3f7ekp5V6nrn7B_M,1691
+reconcile/utils/aws_helper.py,sha256=6Nfgsz0aQ97LBAJ0JBRdnPaFTAkEBSqXvCH6_pVIWdw,2006
 reconcile/utils/binary.py,sha256=3IBnwjKakHM367skPPvG6yVSQYjKt5muQlFNdoa63DU,2352
 reconcile/utils/config.py,sha256=aId5zrPjM_84u_T4yTRE_Psu3zo5-5_JCR6_7Wgv5UQ,990
 reconcile/utils/constants.py,sha256=pOUd97bqZdsAu5RWJ8NUs9cwCY7K9y0eW9VVeJ4fZIU,138
@@ -593,7 +598,7 @@ reconcile/utils/raw_github_api.py,sha256=ZHC-SZuAyRe1zaMoOU7Krt1-zecDxENd9c_NzQY
 reconcile/utils/repo_owners.py,sha256=j-pUjc9PuDzq7KpjNLpnhqfU8tUG4nj2WMhFp4ick7g,6629
 reconcile/utils/ruamel.py,sha256=FzL4_L0FnMOUZmgThrZSMJs5MTdXwiy-E9MZWfk8bh8,397
 reconcile/utils/secret_reader.py,sha256=2DeYAAQFjUULEKlLw3UDAUoND6gbqvCh9uKPtlc-0us,10403
-reconcile/utils/semver_helper.py,sha256=dp86KxjlOc8LHzawMvbxRfZamv7KU7b2SVnZQL-Xg6U,1142
+reconcile/utils/semver_helper.py,sha256=-WfPOMSA2v1h7hT3PwVf-Htg7wOsoKlQC1JdmDX2Ars,1268
 reconcile/utils/sharding.py,sha256=gkYf0lD3IUKQPEmdRJZ70mdDT1c9qWjbdP7evRsUis4,839
 reconcile/utils/slack_api.py,sha256=OPmzU6L9rJx2XXDlZkMlxLjOWu17yC-fVCoUItzQrXw,16295
 reconcile/utils/smtp_client.py,sha256=gJNbBQJpAt5PX4t_TaeNHsXM8vt50bFgndml6yK2b5o,2800
@@ -642,7 +647,7 @@ reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py,sha256=RzEKRT_BhvB2ud9py
 reconcile/utils/mr/user_maintenance.py,sha256=cHPBn8zrReWLHalyk-EFdkFJe9zjVjRoZhT4t2zZfGE,3956
 reconcile/utils/ocm/__init__.py,sha256=5Pcf5cyftDWT5XRi1EzvNklOVxGplJi-v12HN3TDarc,57
 reconcile/utils/ocm/addons.py,sha256=8wVrt16i69KkXq1fQByVheSQRhrRELbuOHb7Tz9bKT0,1675
-reconcile/utils/ocm/base.py,sha256=7jm37vMoIIaSnOZe02_6VZFIh0IDa-VVIqfR1FoddZQ,11872
+reconcile/utils/ocm/base.py,sha256=GclZtCrPkPJmGP9HHvqIlV-8VXSKuaQTQJkA2pklN60,13817
 reconcile/utils/ocm/cluster_groups.py,sha256=F8oqVqN_4QUnGL0K61zZhoYIzJeP57EcmZpwmoV0mr4,1751
 reconcile/utils/ocm/clusters.py,sha256=Q6g5kGSNfxZUZ56LPFAYjOz8xJ2c6QG76V78GvyLxB0,7448
 reconcile/utils/ocm/identity_providers.py,sha256=dKed09N8iWmn39tI_MpwgVe47x23eLsknGbjMUxtwr4,2175
@@ -693,8 +698,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=OvalpVRfY4pNmpMaWHHYqBjV68b1eGQjX8SCyTAXb1w,3501
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc596.dist-info/METADATA,sha256=48NeYqXGwOl1UWs78VYRsxQpABH6M_hfv5xHUEO3cFw,2349
-qontract_reconcile-0.10.1rc596.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qontract_reconcile-0.10.1rc596.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
-qontract_reconcile-0.10.1rc596.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc596.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc597.dist-info/METADATA,sha256=d8mSQupiOAr8Iya1HYE2kxpuoqxKJv1EYTjBdA4WdOs,2349
+qontract_reconcile-0.10.1rc597.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qontract_reconcile-0.10.1rc597.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
+qontract_reconcile-0.10.1rc597.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc597.dist-info/RECORD,,

reconcile/aus/base.py

@@ -19,7 +19,7 @@ from typing import (
 
 import semver
 from croniter import croniter
-from pydantic import BaseModel
+from pydantic import BaseModel, Extra
 from semver import VersionInfo
 
 from reconcile.aus.cluster_version_data import (
@@ -44,6 +44,7 @@ from reconcile.aus.models import (
     OrganizationUpgradeSpec,
     Sector,
 )
+from reconcile.aus.version_gates import HANDLERS
 from reconcile.gql_definitions.advanced_upgrade_service.aus_organization import (
     query as aus_organizations_query,
 )
@@ -71,7 +72,6 @@ from reconcile.utils.ocm.upgrades import (
     create_control_plane_upgrade_policy,
     create_node_pool_upgrade_policy,
     create_upgrade_policy,
-    create_version_agreement,
     delete_addon_upgrade_policy,
     delete_control_plane_upgrade_policy,
     delete_upgrade_policy,
@@ -88,6 +88,7 @@ from reconcile.utils.runtime.integration import (
     QontractReconcileIntegration,
 )
 from reconcile.utils.semver_helper import (
+    get_version_prefix,
     parse_semver,
     sort_versions,
 )
@@ -299,21 +300,6 @@ class AdvancedUpgradeSchedulerBaseIntegration(
             )
 
 
-class GateAgreement(BaseModel):
-    gate: OCMVersionGate
-
-    def create(self, ocm_api: OCMBaseClient, cluster: OCMCluster) -> None:
-        logging.info(
-            f"create agreement for gate {self.gate.id} on cluster {cluster.name} (id={cluster.id})"
-        )
-        agreement = create_version_agreement(ocm_api, self.gate.id, cluster.id)
-        if agreement.get("version_gate") is None:
-            logging.error(
-                "Unexpected response while creating version "
-                f"agreement with id {self.gate.id} for cluster {cluster.name} (id={cluster.id})"
-            )
-
-
 class RemainingSoakDayMetricsBuilder(Protocol):
     def __call__(
         self, cluster_uuid: str, soaking_version: str
@@ -467,18 +453,12 @@ class NodePoolUpgradePolicy(AbstractUpgradePolicy):
         return f"node pool upgrade policy - {remove_none_values_from_dict(details)}"
 
 
-class UpgradePolicyHandler(BaseModel):
+class UpgradePolicyHandler(BaseModel, extra=Extra.forbid):
     """Class to handle upgrade policy actions"""
 
     action: str
     policy: AbstractUpgradePolicy
 
-    gates_to_agree: Optional[list[GateAgreement]]
-
-    def _create_gate_agreements(self, ocm_api: OCMBaseClient) -> None:
-        for gate in self.gates_to_agree or []:
-            gate.create(ocm_api, self.policy.cluster)
-
     def act(self, dry_run: bool, ocm_api: OCMBaseClient) -> None:
         logging.info(f"{self.action} {self.policy.summarize()}")
         if dry_run:
@@ -489,7 +469,6 @@ class UpgradePolicyHandler(BaseModel):
         elif self.action == "delete":
             self.policy.delete(ocm_api)
         elif self.action == "create":
-            self._create_gate_agreements(ocm_api)
             self.policy.create(ocm_api)
 
 
@@ -727,10 +706,34 @@ def gates_for_minor_version(
     return [g for g in gates if g.version_raw_id_prefix == target_version_prefix]
 
 
+def is_gate_applicable_to_cluster(gate: OCMVersionGate, cluster: OCMCluster) -> bool:
+    # check that the cluster has an upgrade path that crosses the gate version
+    minor_version_upgrade_paths = {
+        get_version_prefix(version) for version in cluster.available_upgrades()
+    }
+    if gate.version_raw_id_prefix not in minor_version_upgrade_paths:
+        return False
+
+    # consider only gates after the clusters current minor version
+    # OCM onls supports creating gate agreements for later minor versions than the
+    # current cluster version
+    if not semver.match(
+        f"{cluster.minor_version()}.0", f"<{gate.version_raw_id_prefix}.0"
+    ):
+        return False
+
+    # check the handler for the gate type if it is responsible for this kind
+    # of cluster
+    handler = HANDLERS.get(gate.label)
+    if handler:
+        return handler.gate_applicable_to_cluster(cluster)
+    return False
+
+
 def gates_to_agree(
     gates: list[OCMVersionGate],
     cluster: OCMCluster,
-    ocm_api: OCMBaseClient,
+    acked_gate_ids: set[str],
 ) -> list[OCMVersionGate]:
     """Check via OCM if a version is agreed
 
@@ -742,36 +745,13 @@ def gates_to_agree(
     Returns:
         list[OCMVersionGate]: list of gates a cluster has not agreed on yet
     """
-    applicable_gates = [
-        g
-        for g in gates
-        # todo: sts version gates need special handling - https://issues.redhat.com/browse/APPSRE-7949
-        #       until this is solved, we can't do automated upgrades for STS clusters that cross a version gate
-        #       once we have proper and secure handling get gate agreements for STS clusters, we can use this condition:
-        #       `and (not g.sts_only or g.sts_only == cluster.is_sts())`
-        if not g.sts_only
-        # consider only gates after the clusters current minor version
-        # OCM onls supports creating gate agreements for later minor versions than the
-        # current cluster version
-        and semver.match(
-            f"{cluster.minor_version()}.0", f"<{g.version_raw_id_prefix}.0"
-        )
-    ]
+    applicable_gates = [g for g in gates if is_gate_applicable_to_cluster(g, cluster)]
 
     if applicable_gates:
-        current_agreements = {
-            agreement["version_gate"]["id"]
-            for agreement in get_version_agreement(ocm_api, cluster.id)
-        }
-        return [gate for gate in applicable_gates if gate.id not in current_agreements]
+        return [gate for gate in applicable_gates if gate.id not in acked_gate_ids]
     return []
 
 
-def get_version_prefix(version: str) -> str:
-    semver = parse_semver(version)
-    return f"{semver.major}.{semver.minor}"
-
-
 def upgradeable_version(
     spec: ClusterUpgradeSpec,
     version_data: VersionData,
@@ -1024,18 +1004,24 @@ def calculate_diff(
                         "Skip creation of an upgrade policy."
                     )
                     continue
+                gates = gates_to_agree(
+                    gates=minor_version_gates,
+                    cluster=spec.cluster,
+                    acked_gate_ids={
+                        agreement["version_gate"]["id"]
+                        for agreement in get_version_agreement(ocm_api, spec.cluster.id)
+                    },
+                )
+                if gates:
+                    gate_ids = [gate.id for gate in gates]
+                    logging.debug(
+                        f"[{spec.org.org_id}/{spec.org.name}/{spec.cluster.name}] found gates for {target_version_prefix} - {gate_ids} "
+                        "Skip creation of an upgrade policy until all of them have been acked by the version-gate-approver integration or a user."
+                    )
                 diffs.append(
                     UpgradePolicyHandler(
                         action="create",
                         policy=_create_upgrade_policy(next_schedule, spec, version),
-                        gates_to_agree=[
-                            GateAgreement(gate=g)
-                            for g in gates_to_agree(
-                                minor_version_gates,
-                                spec.cluster,
-                                ocm_api,
-                            )
-                        ],
                     )
                 )
             set_mutex(locked, spec.cluster.id, spec.upgrade_policy.conditions.mutexes)

reconcile/aus/version_gate_approver.py

@@ -0,0 +1,204 @@
+import logging
+from typing import (
+    Callable,
+    Iterable,
+    Optional,
+)
+
+import semver
+
+from reconcile.aus.advanced_upgrade_service import aus_label_key
+from reconcile.aus.base import gates_to_agree, get_orgs_for_environment
+from reconcile.aus.version_gates import ocp_gate_handler, sts_version_gate_handler
+from reconcile.aus.version_gates.handler import GateHandler
+from reconcile.gql_definitions.common.ocm_environments import (
+    query as ocm_environment_query,
+)
+from reconcile.utils import gql
+from reconcile.utils.grouping import group_by
+from reconcile.utils.jobcontroller.controller import (
+    build_job_controller,
+)
+from reconcile.utils.ocm.base import (
+    ClusterDetails,
+    LabelContainer,
+    OCMCluster,
+    OCMVersionGate,
+)
+from reconcile.utils.ocm.clusters import discover_clusters_by_labels
+from reconcile.utils.ocm.search_filters import Filter
+from reconcile.utils.ocm.upgrades import (
+    create_version_agreement,
+    get_version_agreement,
+    get_version_gates,
+)
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+    init_ocm_base_client,
+    init_ocm_base_client_for_org,
+)
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+
+QONTRACT_INTEGRATION = "version-gate-approver"
+QONTRACT_INTEGRATION_VERSION = semver.format_version(0, 1, 0)
+
+
+class VersionGateApproverParams(PydanticRunParams):
+    job_controller_cluster: str
+    job_controller_namespace: str
+    rosa_job_service_account: str
+    rosa_role: str
+    rosa_job_image: Optional[str] = None
+
+
+class VersionGateApprover(QontractReconcileIntegration[VersionGateApproverParams]):
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def initialize_handlers(self, query_func: Callable) -> None:
+        self.handlers: dict[str, GateHandler] = {
+            sts_version_gate_handler.GATE_LABEL: sts_version_gate_handler.STSGateHandler(
+                job_controller=build_job_controller(
+                    integration=QONTRACT_INTEGRATION,
+                    integration_version=QONTRACT_INTEGRATION_VERSION,
+                    cluster=self.params.job_controller_cluster,
+                    namespace=self.params.job_controller_namespace,
+                    secret_reader=self.secret_reader,
+                    dry_run=False,
+                ),
+                aws_iam_role=self.params.rosa_role,
+                rosa_job_service_account=self.params.rosa_job_service_account,
+                rosa_job_image=self.params.rosa_job_image,
+            ),
+            ocp_gate_handler.GATE_LABEL: ocp_gate_handler.OCPGateHandler(),
+        }
+
+    def run(self, dry_run: bool) -> None:
+        gql_api = gql.get_api()
+        self.initialize_handlers(gql_api.query)
+        environments = ocm_environment_query(gql_api.query).environments
+        ocm_apis = {
+            env.name: init_ocm_base_client(env, self.secret_reader)
+            for env in environments
+        }
+        for env in environments:
+            self.process_environment(
+                ocm_env_name=env.name,
+                ocm_api=ocm_apis[env.name],
+                query_func=gql_api.query,
+                dry_run=dry_run,
+            )
+
+    def process_environment(
+        self,
+        ocm_env_name: str,
+        ocm_api: OCMBaseClient,
+        query_func: Callable,
+        dry_run: bool,
+    ) -> None:
+        """
+        Find all clusters with AUS labels in the organization and process them
+        org by org.
+        """
+        # lookup clusters
+        clusters = discover_clusters_by_labels(
+            ocm_api=ocm_api,
+            label_filter=Filter().like("key", aus_label_key("%")),
+        )
+        clusters_by_org_id = group_by(clusters, lambda c: c.organization_id)
+
+        # lookup version gates
+        gates = get_version_gates(ocm_api)
+
+        # lookup organization metadata
+        organizations = get_orgs_for_environment(
+            integration=QONTRACT_INTEGRATION,
+            ocm_env_name=ocm_env_name,
+            query_func=query_func,
+            ocm_organization_ids=set(clusters_by_org_id.keys()),
+        )
+
+        for org in organizations:
+            ocm_org_api = init_ocm_base_client_for_org(org, self.secret_reader)
+            self.process_organization(
+                clusters=clusters_by_org_id.get(org.org_id, []),
+                gates=gates,
+                ocm_api=ocm_org_api,
+                dry_run=dry_run,
+            )
+
+    def process_organization(
+        self,
+        clusters: Iterable[ClusterDetails],
+        gates: list[OCMVersionGate],
+        ocm_api: OCMBaseClient,
+        dry_run: bool,
+    ) -> None:
+        """
+        Process all clusters in an organization.
+        """
+        for cluster in clusters:
+            unacked_gates = gates_to_agree(
+                cluster=cluster.ocm_cluster,
+                gates=gates,
+                acked_gate_ids={
+                    agreement["version_gate"]["id"]
+                    for agreement in get_version_agreement(
+                        ocm_api, cluster.ocm_cluster.id
+                    )
+                },
+            )
+            if not unacked_gates:
+                continue
+            self.process_cluster(
+                cluster=cluster.ocm_cluster,
+                enabled_gate_handlers=get_enabled_gate_handlers(cluster.labels),
+                gates=unacked_gates,
+                ocm_api=ocm_api,
+                ocm_org_id=cluster.organization_id,
+                dry_run=dry_run,
+            )
+
+    def process_cluster(
+        self,
+        cluster: OCMCluster,
+        enabled_gate_handlers: set[str],
+        gates: list[OCMVersionGate],
+        ocm_api: OCMBaseClient,
+        ocm_org_id: str,
+        dry_run: bool,
+    ) -> None:
+        """
+        Process all unacknowledged gates for a cluster.
+        """
+        for gate in gates:
+            if gate.label in self.handlers and gate.label not in enabled_gate_handlers:
+                continue
+            success = self.handlers[gate.label].handle(
+                ocm_api=ocm_api,
+                ocm_org_id=ocm_org_id,
+                cluster=cluster,
+                gate=gate,
+                dry_run=dry_run,
+            )
+            if success and not dry_run:
+                create_version_agreement(ocm_api, gate.id, cluster.id)
+            elif not success:
+                logging.error(
+                    f"Failed to handle gate {gate.id} for cluster {cluster.name}"
+                )
+
+
+def get_enabled_gate_handlers(labels: LabelContainer) -> set[str]:
+    """
+    Get the set of enabled gate handlers from the labels. Default to the OCP
+    gate to keep backwards compatibility (for now).
+    """
+    handler_csv = labels.get_label_value(aus_label_key("version-gate-approvals"))
+    if not handler_csv:
+        return {ocp_gate_handler.GATE_LABEL}
+    return set(handler_csv.split(","))

reconcile/aus/version_gates/__init__.py

@@ -0,0 +1,9 @@
+from typing import Type
+
+from reconcile.aus.version_gates import ocp_gate_handler, sts_version_gate_handler
+from reconcile.aus.version_gates.handler import GateHandler
+
+HANDLERS: dict[str, Type[GateHandler]] = {
+    ocp_gate_handler.GATE_LABEL: ocp_gate_handler.OCPGateHandler,
+    sts_version_gate_handler.GATE_LABEL: sts_version_gate_handler.STSGateHandler,
+}

reconcile/aus/version_gates/handler.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+
+from reconcile.utils.ocm.base import OCMCluster, OCMVersionGate
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+
+class GateHandler(ABC):
+    """
+    A protocol for version gate handlers.
+    """
+
+    @staticmethod
+    @abstractmethod
+    def gate_applicable_to_cluster(cluster: OCMCluster) -> bool:
+        """
+        Check if the gate represented by this handler is applicable for the given cluster.
+        """
+        ...
+
+    @abstractmethod
+    def handle(
+        self,
+        ocm_api: OCMBaseClient,
+        ocm_org_id: str,
+        cluster: OCMCluster,
+        gate: OCMVersionGate,
+        dry_run: bool,
+    ) -> bool:
+        """
+        Take all necessary actions required by a version gate.
+        If successful, return True. Otherwise, return False.
+        """
+        ...

reconcile/aus/version_gates/ocp_gate_handler.py

@@ -0,0 +1,26 @@
+from reconcile.aus.version_gates.handler import GateHandler
+from reconcile.utils.ocm.base import OCMCluster, OCMVersionGate
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+GATE_LABEL = "api.openshift.com/gate-ocp"
+
+
+class OCPGateHandler(GateHandler):
+    """
+    Right now we just ack all gate-ocp gates...
+    We could do better in the future, e.g. inspecting insights findings on the cluster
+    """
+
+    @staticmethod
+    def gate_applicable_to_cluster(_: OCMCluster) -> bool:
+        return True
+
+    def handle(
+        self,
+        ocm_api: OCMBaseClient,
+        ocm_org_id: str,
+        cluster: OCMCluster,
+        gate: OCMVersionGate,
+        dry_run: bool,
+    ) -> bool:
+        return True

reconcile/aus/version_gates/sts_version_gate_handler.py

@@ -0,0 +1,101 @@
+import logging
+from typing import Optional
+
+from reconcile.aus.version_gates.handler import GateHandler
+from reconcile.utils.jobcontroller.controller import K8sJobController
+from reconcile.utils.ocm.base import OCMCluster, OCMVersionGate
+from reconcile.utils.ocm_base_client import OCMBaseClient
+from reconcile.utils.rosa.rosa_cli import RosaCliException
+from reconcile.utils.rosa.session import RosaSession
+
+GATE_LABEL = "api.openshift.com/gate-sts"
+
+
+class STSGateHandler(GateHandler):
+    def __init__(
+        self,
+        job_controller: K8sJobController,
+        aws_iam_role: str,
+        rosa_job_service_account: Optional[str] = None,
+        rosa_job_image: Optional[str] = None,
+    ) -> None:
+        self.job_controller = job_controller
+        self.aws_iam_role = aws_iam_role
+        self.rosa_job_image = rosa_job_image
+        self.rosa_job_service_account = rosa_job_service_account
+
+    @staticmethod
+    def gate_applicable_to_cluster(cluster: OCMCluster) -> bool:
+        """
+        The STS Gate is applicable to all clusters with STS enabled.
+        This could potentially also be OSD STS clusters. While this handler
+        does not handle OSD clusters as of now, it is still important that
+        we report the STS gate to be applicable to OSD STS clusters.
+        """
+        return cluster.is_sts()
+
+    def handle(
+        self,
+        ocm_api: OCMBaseClient,
+        ocm_org_id: str,
+        cluster: OCMCluster,
+        gate: OCMVersionGate,
+        dry_run: bool,
+    ) -> bool:
+        if (
+            not cluster.id
+            or not cluster.aws
+            or not cluster.aws.sts
+            or not cluster.is_sts()
+        ):
+            # checked already but mypy :/
+            return False
+
+        if cluster.is_rosa_hypershift():
+            # thanks to hypershift managed policies, there is nothing to do for us here
+            # returning True will ack the version gate
+            return True
+        if not cluster.is_rosa_classic():
+            # we manage roels only for rosa classic clusters
+            # returning here will prevent OSD STS clusters to be handled right now
+            logging.error(
+                f"Cluster {cluster.id} is not a ROSA cluster. "
+                "STS version gates are only handled for ROSA classic clusters."
+            )
+            return False
+
+        rosa = RosaSession(
+            aws_account_id=cluster.aws.aws_account_id,
+            aws_region=cluster.region.id,
+            aws_iam_role=self.aws_iam_role,
+            ocm_org_id=ocm_org_id,
+            ocm_api=ocm_api,
+            job_controller=self.job_controller,
+            image=self.rosa_job_image,
+            service_account=self.rosa_job_service_account,
+        )
+
+        try:
+            # account role handling
+            account_role_prefix = cluster.aws.account_role_prefix
+            if not account_role_prefix:
+                raise Exception(
+                    f"Can't upgrade account roles. Cluster {cluster.name} does not define spec.aws.account_role_prefix"
+                )
+            rosa.upgrade_account_roles(
+                role_prefix=account_role_prefix,
+                minor_version=gate.version_raw_id_prefix,
+                channel_group=cluster.version.channel_group,
+                dry_run=dry_run,
+            )
+
+            # operator role handling
+            rosa.upgrade_operator_roles(
+                cluster_id=cluster.id,
+                dry_run=dry_run,
+            )
+        except RosaCliException as e:
+            logging.error(f"Failed to upgrade roles for cluster {cluster.name}: {e}")
+            e.write_logs_to_logger(logging.error)
+            return False
+        return True

reconcile/cli.py

@@ -2448,6 +2448,65 @@ def advanced_upgrade_scheduler(
     )
 
 
+@integration.command(short_help="Approves OCM cluster upgrade version gates.")
+@click.option(
+    "--job-controller-cluster",
+    help="The cluster holding the job-controller namepsace",
+    required=True,
+    envvar="JOB_CONTROLLER_CLUSTER",
+)
+@click.option(
+    "--job-controller-namespace",
+    help="The namespace used for ROSA jobs",
+    required=True,
+    envvar="JOB_CONTROLLER_NAMESPACE",
+)
+@click.option(
+    "--rosa-job-service-account",
+    help="The service-account used for ROSA jobs",
+    required=True,
+    envvar="ROSA_JOB_SERVICE_ACCOUNT",
+)
+@click.option(
+    "--rosa-job-image",
+    help="The container image to use to run ROSA cli command jobs",
+    required=False,
+    envvar="ROSA_JOB_IMAGE",
+)
+@click.option(
+    "--rosa-role",
+    help="The role to assume in the ROSA cluster account",
+    required=True,
+    envvar="ROSA_ROLE",
+)
+@click.pass_context
+def version_gate_approver(
+    ctx,
+    job_controller_cluster: str,
+    job_controller_namespace: str,
+    rosa_job_service_account: str,
+    rosa_role: str,
+    rosa_job_image: Optional[str],
+) -> None:
+    from reconcile.aus.version_gate_approver import (
+        VersionGateApprover,
+        VersionGateApproverParams,
+    )
+
+    run_class_integration(
+        integration=VersionGateApprover(
+            VersionGateApproverParams(
+                job_controller_cluster=job_controller_cluster,
+                job_controller_namespace=job_controller_namespace,
+                rosa_job_service_account=rosa_job_service_account,
+                rosa_job_image=rosa_job_image,
+                rosa_role=rosa_role,
+            )
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage Databases and Database Users.")
 @vault_output_path
 @click.pass_context

reconcile/utils/aws_helper.py

@@ -21,6 +21,17 @@ def get_account_uid_from_arn(arn):
     return arn.split(":")[4]
 
 
+def get_role_name_from_arn(arn: str) -> str:
+    # arn:aws:iam::12345:role/role-1 --> role-1
+    return arn.split("/")[-1]
+
+
+def is_aws_managed_resource(arn: str) -> bool:
+    # arn:aws:iam::aws:role/role-1 --> True
+    # arn:aws:iam::12345:role/role-1 --> False
+    return get_account_uid_from_arn(arn) == "aws"
+
+
 def get_details_from_role_link(role_link):
     # https://signin.aws.amazon.com/switchrole?
     # account=<uid>&roleName=<role_name> -->

reconcile/utils/ocm/base.py

@@ -14,6 +14,8 @@ from pydantic import (
     Field,
 )
 
+from reconcile.utils.aws_helper import get_account_uid_from_arn, get_role_name_from_arn
+
 LabelSetTypeVar = TypeVar("LabelSetTypeVar", bound=BaseModel)
 ACTIVE_SUBSCRIPTION_STATES = {"Active", "Reserved"}
 CAPABILITY_MANAGE_CLUSTER_ADMIN = "capability.cluster.manage_cluster_admin"
@@ -47,6 +49,12 @@ class OCMVersionGate(BaseModel):
     id: str
     version_raw_id_prefix: str
     sts_only: bool
+    label: str
+    """
+    the label field holds a readable name for a verion gate, e.g.
+    - api.openshift.com/gate-sts
+    - api.openshift.com/gate-ocp
+    """
 
 
 class OCMClusterGroupId(Enum):
@@ -118,13 +126,62 @@ class OCMClusterFlag(BaseModel):
     enabled: bool
 
 
+class OCMClusterAWSOperatorRole(BaseModel):
+    id: str
+    name: str
+    namespace: str
+    role_arn: str
+    service_account: str
+
+
+class OCMAWSSTS(OCMClusterFlag):
+    role_arn: Optional[str]
+    support_role_arn: Optional[str]
+    oidc_endpoint_url: Optional[str]
+    operator_iam_roles: Optional[list[OCMClusterAWSOperatorRole]]
+    instance_iam_roles: Optional[dict[str, str]]
+    operator_role_prefix: Optional[str]
+
+
 class OCMClusterAWSSettings(BaseModel):
-    sts: Optional[OCMClusterFlag]
+    sts: Optional[OCMAWSSTS]
 
     @property
     def sts_enabled(self) -> bool:
         return self.sts is not None and self.sts.enabled
 
+    @property
+    def aws_account_id(self) -> str:
+        return get_account_uid_from_arn(self.account_roles[0])
+
+    @property
+    def account_roles(self) -> list[str]:
+        if not self.sts or not self.sts.enabled:
+            return []
+        roles = []
+        if self.sts.role_arn:
+            roles.append(self.sts.role_arn)
+        if self.sts.support_role_arn:
+            roles.append(self.sts.support_role_arn)
+        for instance_iam_role in (self.sts.instance_iam_roles or {}).values():
+            roles.append(instance_iam_role)
+        return roles
+
+    @property
+    def account_role_prefix(self) -> Optional[str]:
+        INSTALLER_ROLE_BASE_NAME = "-Installer-Role"
+        installer_role_arn = self.sts.role_arn if self.sts else None
+        if installer_role_arn and installer_role_arn.endswith(INSTALLER_ROLE_BASE_NAME):
+            installer_role_name = get_role_name_from_arn(installer_role_arn)
+            return installer_role_name.removesuffix(INSTALLER_ROLE_BASE_NAME)
+        return None
+
+    @property
+    def operator_roles(self) -> list[str]:
+        if not self.sts:
+            return []
+        return [role.role_arn for role in self.sts.operator_iam_roles or []]
+
 
 class OCMClusterVersion(BaseModel):
     id: str
@@ -147,7 +204,6 @@ class OCMClusterDns(BaseModel):
 
 
 class OCMExternalConfiguration(BaseModel):
-    kind: str
     syncsets: dict
 
 
@@ -199,6 +255,9 @@ class OCMCluster(BaseModel):
     def is_osd(self) -> bool:
         return self.product.id == PRODUCT_ID_OSD
 
+    def is_rosa(self) -> bool:
+        return self.product.id == PRODUCT_ID_ROSA
+
     def is_rosa_classic(self) -> bool:
         return self.product.id == PRODUCT_ID_ROSA and not self.hypershift.enabled
 

reconcile/utils/semver_helper.py

@@ -35,3 +35,8 @@ def sort_versions(versions: Iterable[str]) -> list[str]:
 
 def is_version_bumped(current_version: str, previous_version: str) -> bool:
     return parse_semver(current_version) > parse_semver(previous_version)
+
+
+def get_version_prefix(version: str) -> str:
+    semver = parse_semver(version)
+    return f"{semver.major}.{semver.minor}"