qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

tools/qontract_cli.py

@@ -1,31 +1,40 @@
 #!/usr/bin/env python3
+# ruff: noqa: PLC0415 - `import` should be at the top-level of a file
 
 import base64
 import json
+import logging
 import os
 import re
 import sys
+import tempfile
+import textwrap
 from collections import defaultdict
-from datetime import datetime
-from operator import itemgetter
-from typing import (
-    Any,
-    Optional,
+from datetime import (
+    UTC,
+    datetime,
+    timedelta,
 )
+from operator import itemgetter
+from pathlib import Path
+from statistics import median
+from textwrap import dedent
+from typing import Any
 
+import boto3
 import click
+import click.core
 import requests
 import yaml
 from rich import box
-from rich.console import (
-    Console,
-    Group,
-)
+from rich import print as rich_print
+from rich.console import Console, Group
+from rich.prompt import Confirm
 from rich.table import Table
 from rich.tree import Tree
-from sretoolbox.utils import threaded
 
 import reconcile.aus.base as aus
+import reconcile.change_owners.change_log_tracking as cl
 import reconcile.openshift_base as ob
 import reconcile.openshift_resources_base as orb
 import reconcile.prometheus_rules_tester.integration as ptr
@@ -34,31 +43,59 @@ import reconcile.terraform_users as tfu
 import reconcile.terraform_vpc_peerings as tfvpc
 from reconcile import queries
 from reconcile.aus.base import (
+    AbstractUpgradePolicy,
     AdvancedUpgradeSchedulerBaseIntegration,
     AdvancedUpgradeSchedulerBaseIntegrationParams,
+    addon_upgrade_policy_soonest_next_run,
+    init_addon_service_version,
 )
+from reconcile.aus.models import OrganizationUpgradeSpec
 from reconcile.change_owners.bundle import NoOpFileDiffResolver
+from reconcile.change_owners.change_log_tracking import (
+    BUNDLE_DIFFS_OBJ,
+    ChangeLog,
+    ChangeLogItem,
+)
 from reconcile.change_owners.change_owners import (
     fetch_change_type_processors,
     fetch_self_service_roles,
 )
 from reconcile.checkpoint import report_invalid_metadata
 from reconcile.cli import (
+    TERRAFORM_VERSION,
+    TERRAFORM_VERSION_REGEX,
+    cluster_name,
     config_file,
+    namespace_name,
     use_jump_host,
 )
+from reconcile.cli import (
+    threaded as thread_pool_size,
+)
+from reconcile.gql_definitions.advanced_upgrade_service.aus_clusters import (
+    query as aus_clusters_query,
+)
 from reconcile.gql_definitions.common.app_interface_vault_settings import (
     AppInterfaceSettingsV1,
 )
+from reconcile.gql_definitions.fragments.aus_organization import AUSOCMOrganization
+from reconcile.gql_definitions.integrations import integrations as integrations_gql
+from reconcile.gql_definitions.maintenance import maintenances as maintenances_gql
 from reconcile.jenkins_job_builder import init_jjb
-from reconcile.prometheus_rules_tester_old import get_data_from_jinja_test_template
 from reconcile.slack_base import slackapi_from_queries
+from reconcile.status_board import StatusBoardExporterIntegration
 from reconcile.typed_queries.alerting_services_settings import get_alerting_services
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
+from reconcile.typed_queries.app_quay_repos_escalation_policies import (
+    get_apps_quay_repos_escalation_policies,
+)
 from reconcile.typed_queries.clusters import get_clusters
 from reconcile.typed_queries.saas_files import get_saas_files
+from reconcile.typed_queries.slo_documents import get_slo_documents
+from reconcile.typed_queries.status_board import get_status_board
 from reconcile.utils import (
     amtool,
     config,
@@ -67,8 +104,18 @@ from reconcile.utils import (
     promtool,
 )
 from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.cluster_version_data import VersionData
+from reconcile.utils.binary import (
+    binary,
+    binary_version,
+)
+from reconcile.utils.early_exit_cache import (
+    CacheKey,
+    CacheKeyWithDigest,
+    CacheValue,
+    EarlyExitCache,
+)
 from reconcile.utils.environ import environ
+from reconcile.utils.external_resource_spec import ExternalResourceSpec
 from reconcile.utils.external_resources import (
     PROVIDER_AWS,
     get_external_resource_specs,
@@ -79,18 +126,29 @@ from reconcile.utils.gitlab_api import (
     MRState,
     MRStatus,
 )
+from reconcile.utils.gql import GqlApiSingleton
 from reconcile.utils.jjb_client import JJB
+from reconcile.utils.keycloak import (
+    KeycloakAPI,
+    SSOClient,
+)
 from reconcile.utils.mr.labels import (
+    AVS,
     SAAS_FILE_UPDATE,
     SELF_SERVICEABLE,
+    SHOW_SELF_SERVICEABLE_IN_REVIEW_QUEUE,
 )
 from reconcile.utils.oc import (
     OC_Map,
     OCLogMsg,
 )
-from reconcile.utils.oc_map import init_oc_map_from_clusters
-from reconcile.utils.ocm import OCMMap
+from reconcile.utils.oc_map import (
+    init_oc_map_from_clusters,
+)
+from reconcile.utils.ocm import OCM_PRODUCT_ROSA, OCMMap
+from reconcile.utils.ocm_base_client import init_ocm_base_client
 from reconcile.utils.output import print_output
+from reconcile.utils.saasherder.models import TargetSpec
 from reconcile.utils.saasherder.saasherder import SaasHerder
 from reconcile.utils.secret_reader import (
     SecretReader,
@@ -99,10 +157,22 @@ from reconcile.utils.secret_reader import (
 from reconcile.utils.semver_helper import parse_semver
 from reconcile.utils.state import init_state
 from reconcile.utils.terraform_client import TerraformClient as Terraform
+from tools.cli_commands.cost_report.aws import AwsCostReportCommand
+from tools.cli_commands.cost_report.openshift import OpenShiftCostReportCommand
+from tools.cli_commands.cost_report.openshift_cost_optimization import (
+    OpenShiftCostOptimizationReportCommand,
+)
+from tools.cli_commands.erv2 import (
+    Erv2Cli,
+    TerraformCli,
+    progress_spinner,
+    task,
+)
 from tools.cli_commands.gpg_encrypt import (
     GPGEncryptCommand,
     GPGEncryptCommandData,
 )
+from tools.cli_commands.systems_and_tools import get_systems_and_tools_inventory
 from tools.sre_checkpoints import (
     full_name,
     get_latest_sre_checkpoints,
@@ -143,6 +213,11 @@ def root(ctx, configfile):
     gql.init_from_config()
 
 
+@root.result_callback()
+def exit_cli(ctx, configfile):
+    GqlApiSingleton.close()
+
+
 @root.group()
 @output
 @sort
@@ -258,26 +333,26 @@ def cluster_upgrades(ctx, name):
 def version_history(ctx):
     import reconcile.aus.ocm_upgrade_scheduler as ous
 
-    settings = queries.get_app_interface_settings()
-    clusters = queries.get_clusters()
-    clusters = [c for c in clusters if c.get("upgradePolicy") is not None]
-    ocm_map = OCMMap(clusters=clusters, settings=settings)
-
-    version_data_map = aus.get_version_data_map(
-        dry_run=True,
-        upgrade_policies=[],
-        ocm_map=ocm_map,
-        integration=ous.QONTRACT_INTEGRATION,
-    )
+    clusters = aus_clusters_query(query_func=gql.get_api().query).clusters or []
+    orgs = {
+        c.ocm.org_id: OrganizationUpgradeSpec(org=c.ocm, specs=[])
+        for c in clusters
+        if c.ocm and c.upgrade_policy
+    }
 
     results = []
-    for ocm_name, version_data in version_data_map.items():
+    for org_spec in orgs.values():
+        version_data = aus.get_version_data_map(
+            dry_run=True,
+            org_upgrade_spec=org_spec,
+            integration=ous.QONTRACT_INTEGRATION,
+        ).get(org_spec.org.environment.name, org_spec.org.org_id)
         for version, version_history in version_data.versions.items():
             if not version:
                 continue
             for workload, workload_data in version_history.workloads.items():
                 item = {
-                    "ocm": ocm_name,
+                    "ocm": f"{org_spec.org.environment.name}/{org_spec.org.org_id}",
                     "version": parse_semver(version),
                     "workload": workload,
                     "soak_days": round(workload_data.soak_days, 2),
@@ -289,51 +364,33 @@ def version_history(ctx):
     print_output(ctx.obj["options"], results, columns)
 
 
-def soaking_days(
-    version_data_map: dict[str, VersionData],
-    upgrades: list[str],
-    workload: str,
-    only_soaking: bool,
-) -> dict[str, float]:
-    soaking = {}
-    for version in upgrades:
-        for h in version_data_map.values():
-            workload_history = h.workload_history(version, workload)
-            soaking[version] = round(workload_history.soak_days, 2)
-        if not only_soaking and version not in soaking:
-            soaking[version] = 0
-    return soaking
-
-
 def get_upgrade_policies_data(
-    clusters,
+    org_upgrade_specs: list[OrganizationUpgradeSpec],
     md_output,
     integration,
     workload=None,
     show_only_soaking_upgrades=False,
     by_workload=False,
-):
-    if not clusters:
+) -> list:
+    if not org_upgrade_specs:
         return []
 
-    settings = queries.get_app_interface_settings()
-    ocm_map = OCMMap(
-        clusters=clusters,
-        settings=settings,
-        init_version_gates=True,
-    )
-    current_state = aus.fetch_current_state(clusters, ocm_map)
-    desired_state = aus.fetch_desired_state(clusters, ocm_map)
-
-    version_data_map = aus.get_version_data_map(
-        dry_run=True, upgrade_policies=[], ocm_map=ocm_map, integration=integration
-    )
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
     results = []
 
-    def soaking_str(soaking, upgrade_policy, upgradeable_version):
-        upgrade_version = upgrade_policy.get("version")
-        upgrade_next_run = upgrade_policy.get("next_run")
+    def soaking_str(
+        soaking: dict[str, Any],
+        upgrade_policy: AbstractUpgradePolicy | None,
+        upgradeable_version: str | None,
+    ) -> str:
+        if upgrade_policy:
+            upgrade_version = upgrade_policy.version
+            upgrade_next_run = upgrade_policy.next_run
+        else:
+            upgrade_version = None
+            upgrade_next_run = None
         upgrade_emoji = "💫"
         if upgrade_next_run:
             dt = datetime.strptime(upgrade_next_run, "%Y-%m-%dT%H:%M:%SZ")
@@ -353,88 +410,96 @@ def get_upgrade_policies_data(
                     sorted_soaking[i] = (v, f"{s} 🎉")
         return ", ".join([f"{v} ({s})" for v, s in sorted_soaking])
 
-    for c in desired_state:
-        cluster_name, version = c["cluster"], c["current_version"]
-        channel, schedule = c["channel"], c.get("schedule")
-        soakdays = c.get("conditions", {}).get("soakDays")
-        mutexes = c.get("conditions", {}).get("mutexes") or []
-        sector = ""
-        if c.get("conditions", {}).get("sector"):
-            sector = c["conditions"]["sector"].name
-        ocm_org = ocm_map.get(cluster_name)
-        ocm_spec = ocm_org.clusters[cluster_name]
-        item = {
-            "ocm": ocm_org.name,
-            "cluster": cluster_name,
-            "id": ocm_spec.spec.id,
-            "api": ocm_spec.server_url,
-            "console": ocm_spec.console_url,
-            "domain": ocm_spec.domain,
-            "version": version,
-            "channel": channel,
-            "schedule": schedule,
-            "sector": sector,
-            "soak_days": soakdays,
-            "mutexes": ", ".join(mutexes),
-        }
+    for org_spec in org_upgrade_specs:
+        ocm_api = init_ocm_base_client(org_spec.org.environment, secret_reader)
+        current_state = aus.fetch_current_state(ocm_api, org_spec)
 
-        if "workloads" not in c:
-            results.append(item)
-            continue
+        version_data = aus.get_version_data_map(
+            dry_run=True,
+            org_upgrade_spec=org_spec,
+            integration=integration,
+        ).get(org_spec.org.environment.name, org_spec.org.org_id)
 
-        upgrades = [
-            u
-            for u in c.get("available_upgrades") or []
-            if not ocm_org.version_blocked(u)
-        ]
+        for upgrade_spec in org_spec.specs:
+            cluster = upgrade_spec.cluster
+            item = {
+                "ocm": org_spec.org.name,
+                "cluster": cluster.name,
+                "id": cluster.id,
+                "api": cluster.api_url,
+                "console": cluster.console_url,
+                "domain": cluster.base_domain,
+                "version": cluster.version.raw_id,
+                "channel": cluster.version.channel_group,
+                "schedule": upgrade_spec.upgrade_policy.schedule,
+                "sector": upgrade_spec.upgrade_policy.conditions.sector or "",
+                "soak_days": upgrade_spec.upgrade_policy.conditions.soak_days,
+                "mutexes": ", ".join(
+                    upgrade_spec.upgrade_policy.conditions.mutexes or []
+                ),
+            }
 
-        current = [c for c in current_state if c["cluster"] == cluster_name]
-        upgrade_policy = {}
-        if current and current[0]["schedule_type"] == "manual":
-            upgrade_policy = current[0]
+            if not upgrade_spec.upgrade_policy.workloads:
+                results.append(item)
+                continue
 
-        upgradeable_version = aus.upgradeable_version(
-            c, version_data_map, ocm_org, upgrades
-        )
+            upgrades = [
+                u
+                for u in cluster.available_upgrades()
+                if not upgrade_spec.version_blocked(u)
+            ]
+
+            current = [c for c in current_state if c.cluster.name == cluster.name]
+            upgrade_policy = None
+            if current and current[0].schedule_type == "manual":
+                upgrade_policy = current[0]
+
+            sector = (
+                org_spec.sectors.get(upgrade_spec.upgrade_policy.conditions.sector)
+                if upgrade_spec.upgrade_policy.conditions.sector
+                else None
+            )
+            upgradeable_version = aus.upgradeable_version(
+                upgrade_spec, version_data, sector
+            )
 
-        workload_soaking_upgrades = {}
-        for w in c.get("workloads", []):
-            if not workload or workload == w:
-                s = soaking_days(
-                    version_data_map, upgrades, w, show_only_soaking_upgrades
-                )
-                workload_soaking_upgrades[w] = s
+            workload_soaking_upgrades = {}
+            for w in upgrade_spec.upgrade_policy.workloads:
+                if not workload or workload == w:
+                    s = aus.soaking_days(
+                        version_data,
+                        upgrades,
+                        w,
+                        show_only_soaking_upgrades,
+                    )
+                    workload_soaking_upgrades[w] = s
 
-        if by_workload:
-            for w, soaking in workload_soaking_upgrades.items():
-                i = item.copy()
-                i.update(
-                    {
+            if by_workload:
+                for w, soaking in workload_soaking_upgrades.items():
+                    i = item.copy()
+                    i.update({
                         "workload": w,
                         "soaking_upgrades": soaking_str(
                             soaking, upgrade_policy, upgradeable_version
                         ),
-                    }
-                )
-                results.append(i)
-        else:
-            workloads = sorted(c.get("workloads", []))
-            w = ", ".join(workloads)
-            soaking = {}
-            for v in upgrades:
-                soaks = [s.get(v, 0) for s in workload_soaking_upgrades.values()]
-                min_soaks = min(soaks)
-                if not show_only_soaking_upgrades or min_soaks > 0:
-                    soaking[v] = min_soaks
-            item.update(
-                {
+                    })
+                    results.append(i)
+            else:
+                workloads = sorted(upgrade_spec.upgrade_policy.workloads)
+                w = ", ".join(workloads)
+                soaking = {}
+                for v in upgrades:
+                    soaks = [s.get(v, 0) for s in workload_soaking_upgrades.values()]
+                    min_soaks = min(soaks)
+                    if not show_only_soaking_upgrades or min_soaks > 0:
+                        soaking[v] = min_soaks
+                item.update({
                     "workload": w,
                     "soaking_upgrades": soaking_str(
                         soaking, upgrade_policy, upgradeable_version
                     ),
-                }
-            )
-            results.append(item)
+                })
+                results.append(item)
 
     return results
 
@@ -492,109 +557,15 @@ def cluster_upgrade_policies(
     show_only_soaking_upgrades=False,
     by_workload=False,
 ):
-    from reconcile.aus.ocm_upgrade_scheduler import (
-        OCMClusterUpgradeSchedulerIntegration,
-    )
-
-    integration = OCMClusterUpgradeSchedulerIntegration(
-        AdvancedUpgradeSchedulerBaseIntegrationParams()
-    )
-    generate_cluster_upgrade_policies_report(
-        ctx,
-        integration=integration,
-        cluster=cluster,
-        workload=workload,
-        show_only_soaking_upgrades=show_only_soaking_upgrades,
-        by_workload=by_workload,
-    )
-
-
-def generate_cluster_upgrade_policies_report(
-    ctx,
-    integration: AdvancedUpgradeSchedulerBaseIntegration,
-    cluster: Optional[str],
-    workload: Optional[str],
-    show_only_soaking_upgrades: bool,
-    by_workload: bool,
-) -> None:
-    md_output = ctx.obj["options"]["output"] == "md"
-
-    upgrade_specs = integration.get_upgrade_specs()
-    clusters = [
-        s.dict(by_alias=True)
-        for org_upgrade_specs in upgrade_specs.values()
-        for org_upgrade_spec in org_upgrade_specs.values()
-        for s in org_upgrade_spec.specs
-    ]
-
-    if cluster:
-        clusters = [c for c in clusters if cluster == c["name"]]
-    if workload:
-        clusters = [
-            c for c in clusters if workload in c["upgradePolicy"].get("workloads", [])
-        ]
-
-    results = get_upgrade_policies_data(
-        clusters,
-        md_output,
-        integration.name,
-        workload,
-        show_only_soaking_upgrades,
-        by_workload,
+    print(
+        "https://grafana.app-sre.devshift.net/d/ukLXCSwVz/aus-cluster-upgrade-overview"
     )
 
-    if md_output:
-        fields = [
-            {"key": "cluster", "sortable": True},
-            {"key": "version", "sortable": True},
-            {"key": "channel", "sortable": True},
-            {"key": "schedule"},
-            {"key": "sector", "sortable": True},
-            {"key": "mutexes", "sortable": True},
-            {"key": "soak_days", "sortable": True},
-            {"key": "workload"},
-            {"key": "soaking_upgrades"},
-        ]
-        md = """
-{}
-
-```json:table
-{}
-```
-        """
-        md = md.format(
-            upgrade_policies_output_description,
-            json.dumps(
-                {"fields": fields, "items": results, "filter": True, "caption": ""},
-                indent=1,
-            ),
-        )
-        print(md)
-    else:
-        columns = [
-            "cluster",
-            "version",
-            "channel",
-            "schedule",
-            "sector",
-            "mutexes",
-            "soak_days",
-            "workload",
-            "soaking_upgrades",
-        ]
-        ctx.obj["options"]["to_string"] = True
-        print_output(ctx.obj["options"], results, columns)
-
 
-def inherit_version_data_text(ocm_org: str, ocm_specs: list[dict]) -> str:
-    ocm_specs_for_org = [o for o in ocm_specs if o["name"] == ocm_org]
-    if not ocm_specs_for_org:
-        raise ValueError(f"{ocm_org} not found in list of organizations")
-    ocm_spec = ocm_specs_for_org[0]
-    inherit_version_data = ocm_spec["inheritVersionData"]
-    if not inherit_version_data:
+def inherit_version_data_text(org: AUSOCMOrganization) -> str:
+    if not org.inherit_version_data:
         return ""
-    inherited_orgs = [f"[{o['name']}](#{o['name']})" for o in inherit_version_data]
+    inherited_orgs = [f"[{o.name}](#{o.name})" for o in org.inherit_version_data]
     return f"inheriting version data from {', '.join(inherited_orgs)}"
 
 
@@ -616,16 +587,37 @@ def ocm_fleet_upgrade_policies(
 
 
 @get.command()
+@click.option(
+    "--ocm-env",
+    help="The OCM environment AUS should operator on. If none is specified, all environments will be operated on.",
+    required=False,
+    envvar="AUS_OCM_ENV",
+)
+@click.option(
+    "--ocm-org-ids",
+    help="A comma seperated list of OCM organization IDs AUS should operator on. If none is specified, all organizations are considered.",
+    required=False,
+    envvar="AUS_OCM_ORG_IDS",
+)
+@click.option(
+    "--ignore-sts-clusters",
+    is_flag=True,
+    default=os.environ.get("IGNORE_STS_CLUSTERS", False),
+    help="Ignore STS clusters",
+)
 @click.pass_context
-def aus_fleet_upgrade_policies(
-    ctx,
-):
+def aus_fleet_upgrade_policies(ctx, ocm_env, ocm_org_ids, ignore_sts_clusters):
     from reconcile.aus.advanced_upgrade_service import AdvancedUpgradeServiceIntegration
 
+    parsed_ocm_org_ids = set(ocm_org_ids.split(",")) if ocm_org_ids else None
     generate_fleet_upgrade_policices_report(
         ctx,
         AdvancedUpgradeServiceIntegration(
-            AdvancedUpgradeSchedulerBaseIntegrationParams()
+            AdvancedUpgradeSchedulerBaseIntegrationParams(
+                ocm_environment=ocm_env,
+                ocm_organization_ids=parsed_ocm_org_ids,
+                ignore_sts_clusters=ignore_sts_clusters,
+            )
         ),
     )
 
@@ -633,25 +625,18 @@ def aus_fleet_upgrade_policies(
 def generate_fleet_upgrade_policices_report(
     ctx, aus_integration: AdvancedUpgradeSchedulerBaseIntegration
 ):
-
     md_output = ctx.obj["options"]["output"] == "md"
 
-    upgrade_specs = aus_integration.get_upgrade_specs()
-    upgrade_policies = [
-        s.dict(by_alias=True)
-        for org_upgrade_specs in upgrade_specs.values()
-        for org_upgrade_spec in org_upgrade_specs.values()
-        for s in org_upgrade_spec.specs
-    ]
-
-    ocm_org_specs = [
-        org_upgrade_spec.org.dict(by_alias=True)
-        for org_upgrade_specs in upgrade_specs.values()
-        for org_upgrade_spec in org_upgrade_specs.values()
-    ]
+    org_upgrade_specs: dict[str, OrganizationUpgradeSpec] = {}
+    for orgs in aus_integration.get_upgrade_specs().values():
+        for org_spec in orgs.values():
+            if org_spec.specs:
+                org_upgrade_specs[org_spec.org.name] = org_spec
 
     results = get_upgrade_policies_data(
-        upgrade_policies, md_output, integration=aus_integration.name
+        list(org_upgrade_specs.values()),
+        md_output,
+        aus_integration.name,
     )
 
     if md_output:
@@ -684,7 +669,7 @@ def generate_fleet_upgrade_policices_report(
             print(
                 ocm_org_section.format(
                     ocm_org,
-                    inherit_version_data_text(ocm_org, ocm_org_specs),
+                    inherit_version_data_text(org_upgrade_specs[ocm_org].org),
                     json_data,
                 )
             )
@@ -708,9 +693,9 @@ def generate_fleet_upgrade_policices_report(
 
 @get.command()
 @click.pass_context
-def ocm_addon_upgrade_policies(ctx):
-
+def ocm_addon_upgrade_policies(ctx: click.core.Context) -> None:
     import reconcile.aus.ocm_addons_upgrade_scheduler_org as oauso
+    from reconcile.aus.models import ClusterAddonUpgradeSpec
 
     integration = oauso.OCMAddonsUpgradeSchedulerOrgIntegration(
         AdvancedUpgradeSchedulerBaseIntegrationParams()
@@ -721,39 +706,36 @@ def ocm_addon_upgrade_policies(ctx):
         print("We only support md output for now")
         sys.exit(1)
 
-    upgrade_specs = integration.get_upgrade_specs()
+    org_upgrade_specs: dict[str, OrganizationUpgradeSpec] = {}
+    for orgs in integration.get_upgrade_specs().values():
+        for org_spec in orgs.values():
+            if org_spec.specs:
+                org_upgrade_specs[org_spec.org.name] = org_spec
+
+    output: dict[str, list] = {}
+
+    for org_upgrade_spec in org_upgrade_specs.values():
+        ocm_output = output.setdefault(org_upgrade_spec.org.name, [])
+        for spec in org_upgrade_spec.specs:
+            if isinstance(spec, ClusterAddonUpgradeSpec):
+                available_upgrades = spec.get_available_upgrades()
+                next_version = (
+                    available_upgrades[-1] if len(available_upgrades) > 0 else ""
+                )
+                ocm_output.append({
+                    "cluster": spec.cluster.name,
+                    "addon_id": spec.addon.id,
+                    "current_version": spec.current_version,
+                    "schedule": spec.upgrade_policy.schedule,
+                    "sector": spec.upgrade_policy.conditions.sector,
+                    "mutexes": ", ".join(spec.upgrade_policy.conditions.mutexes or []),
+                    "soak_days": spec.upgrade_policy.conditions.soak_days,
+                    "workloads": ", ".join(spec.upgrade_policy.workloads),
+                    "next_version": next_version
+                    if next_version != spec.current_version
+                    else "",
+                })
 
-    output = {}
-    for upgrade_policies_per_org in upgrade_specs.values():
-        for org_name, org_spec in upgrade_policies_per_org.items():
-            ocm_map, addon_states = oauso.get_state_for_org_spec_per_addon(
-                org_spec, fetch_current_state=False
-            )
-            ocm = ocm_map[org_name]
-            for addon_state in addon_states:
-                next_version = ocm.get_addon_version(addon_state.addon_id)
-                ocm_output = output.setdefault(org_name, [])
-                for d in addon_state.desired_state:
-                    sector = ""
-                    conditions = d.get("conditions") or {}
-                    if conditions.get("sector"):
-                        sector = conditions["sector"].name
-                    version = d["current_version"]
-                    ocm_output.append(
-                        {
-                            "cluster": d["cluster"],
-                            "addon_id": addon_state.addon_id,
-                            "current_version": version,
-                            "schedule": d["schedule"],
-                            "sector": sector,
-                            "mutexes": ", ".join(conditions.get("mutexes") or []),
-                            "soak_days": conditions.get("soakDays"),
-                            "workloads": ", ".join(d["workloads"]),
-                            "next_version": next_version
-                            if next_version != version
-                            else "",
-                        }
-                    )
     fields = [
         {"key": "cluster", "sortable": True},
         {"key": "addon_id", "sortable": True},
@@ -772,11 +754,6 @@ def ocm_addon_upgrade_policies(ctx):
 {}
 ```
     """
-    ocm_org_specs = [
-        org_upgrade_spec.org.dict(by_alias=True)
-        for org_upgrade_specs in upgrade_specs.values()
-        for org_upgrade_spec in org_upgrade_specs.values()
-    ]
     for ocm_name in sorted(output.keys()):
         json_data = json.dumps(
             {
@@ -789,9 +766,96 @@ def ocm_addon_upgrade_policies(ctx):
         )
         print(
             section.format(
-                ocm_name, inherit_version_data_text(ocm_name, ocm_org_specs), json_data
+                ocm_name,
+                inherit_version_data_text(org_upgrade_specs[ocm_name].org),
+                json_data,
+            )
+        )
+
+
+@get.command()
+@click.option(
+    "--days",
+    help="Days to consider for the report. Cannot be used with timestamp options.",
+    type=int,
+)
+@click.option(
+    "--from-timestamp",
+    help="Specifies starting Unix time to consider in the report. It requires "
+    "--to-timestamp to be set. It cannot be used with --days option",
+    type=int,
+)
+@click.option(
+    "--to-timestamp",
+    help="Specifies ending Unix time to consider in the report. It requires "
+    "--from-timestamp to be set. It cannot be used with --days option",
+    type=int,
+)
+@click.pass_context
+def sd_app_sre_alert_report(
+    ctx: click.core.Context,
+    days: int | None,
+    from_timestamp: int | None,
+    to_timestamp: int | None,
+) -> None:
+    import tools.sd_app_sre_alert_report as report
+
+    if days:
+        if from_timestamp or to_timestamp:
+            print(
+                "Please don't specify --days or --from-timestamp and --to_timestamp "
+                "options at the same time"
             )
+            sys.exit(1)
+
+        now = datetime.utcnow()
+        from_timestamp = int((now - timedelta(days=days)).timestamp())
+        to_timestamp = int(now.timestamp())
+
+    if not days:
+        if not (from_timestamp and to_timestamp):
+            print(
+                "Please specify --from-timestamp and --to-timestamp options if --days "
+                "is not set"
+            )
+            sys.exit(1)
+
+    slack = slackapi_from_queries(
+        integration_name=report.QONTRACT_INTEGRATION, init_usergroups=False
+    )
+    alerts = report.group_alerts(
+        slack.get_flat_conversation_history(
+            from_timestamp=from_timestamp,  # type: ignore[arg-type]
+            to_timestamp=to_timestamp,
         )
+    )
+    alert_stats = report.gen_alert_stats(alerts)
+
+    columns = [
+        "Alert name",
+        "Triggered",
+        "Resolved",
+        "Median time to resolve (h:mm:ss)",
+    ]
+    table_data: list[dict[str, str]] = []
+    for alert_name, data in sorted(
+        alert_stats.items(), key=lambda i: i[1].triggered_alerts, reverse=True
+    ):
+        median_elapsed = ""
+        if data.elapsed_times:
+            seconds = round(median(data.elapsed_times))
+            median_elapsed = str(timedelta(seconds=seconds))
+
+        table_data.append({
+            "Alert name": alert_name,
+            "Triggered": str(data.triggered_alerts),
+            "Resolved": str(data.resolved_alerts),
+            "Median time to resolve (h:mm:ss)": median_elapsed,
+        })
+
+    # TODO(mafriedm, rporres): Fix this
+    ctx.obj["options"]["sort"] = False
+    print_output(ctx.obj["options"], table_data, columns)
 
 
 @root.command()
@@ -861,14 +925,27 @@ def upgrade_cluster_addon(
         )
     print(["create", ocm_org, cluster, addon, ocm_addon_version])
     if not dry_run:
-        spec = {
-            "version": ocm_addon_version,
-            "schedule_type": "manual",
-            "addon_id": addon,
-            "cluster_id": ocm.cluster_ids[cluster],
-            "upgrade_type": "ADDON",
-        }
-        ocm.create_addon_upgrade_policy(cluster, spec)
+        # detection addon service version
+        ocm_env_labels = json.loads(ocm_info["environment"].get("labels") or "{}")
+        addon_service_version = (
+            ocm_env_labels.get("feature_flag_addon_service_version") or "v2"
+        )
+        addon_service = init_addon_service_version(addon_service_version)
+
+        addon_service.create_addon_upgrade_policy(
+            ocm_api=ocm._ocm_client,
+            cluster_id=ocm.cluster_ids[cluster],
+            addon_id=ocm_addon["id"],
+            schedule_type="manual",
+            version=ocm_addon_version,
+            next_run=addon_upgrade_policy_soonest_next_run(),
+        )
+
+
+def has_cluster_account_access(cluster: dict[str, Any]):
+    spec = cluster.get("spec") or {}
+    account = spec.get("account")
+    return account or cluster.get("awsInfrastructureManagementAccounts") is not None
 
 
 @get.command()
@@ -879,8 +956,7 @@ def clusters_network(ctx, name):
     clusters = [
         c
         for c in queries.get_clusters()
-        if c.get("ocm") is not None
-        and c.get("awsInfrastructureManagementAccounts") is not None
+        if c.get("ocm") is not None and has_cluster_account_access(c)
     ]
     if name:
         clusters = [c for c in clusters if c["name"] == name]
@@ -897,17 +973,36 @@ def clusters_network(ctx, name):
 
     for cluster in clusters:
         cluster_name = cluster["name"]
+        product = cluster.get("spec", {}).get("product", "")
         management_account = tfvpc._get_default_management_account(cluster)
-        account = tfvpc._build_infrastructure_assume_role(
-            management_account, cluster, ocm_map.get(cluster_name)
-        )
-        if not account:
-            continue
-        account["resourcesDefaultRegion"] = management_account["resourcesDefaultRegion"]
+
+        # we shouldn't need to check if cluster product is ROSA, but currently to make
+        # accepter side work in a cluster-vpc peering we need to define the
+        # awsInfrastructureManagementAccounts, that make management_account not None
+        # See https://issues.redhat.com/browse/APPSRE-8224
+        if management_account is None or product == "rosa":
+            # This is a CCS/ROSA cluster.
+            # We can access the account directly, without assuming a network-mgmt role
+            account = cluster["spec"]["account"]
+            account.update({
+                "assume_role": "",
+                "assume_region": cluster["spec"]["region"],
+                "assume_cidr": cluster["network"]["vpc"],
+            })
+        else:
+            account = tfvpc._build_infrastructure_assume_role(
+                management_account,
+                cluster,
+                ocm_map.get(cluster_name),
+                provided_assume_role=None,
+            )
+            account["resourcesDefaultRegion"] = management_account[
+                "resourcesDefaultRegion"
+            ]
         with AWSApi(1, [account], settings=settings, init_users=False) as aws_api:
-            vpc_id, _, _ = aws_api.get_cluster_vpc_details(account)
+            vpc_id, _, _, _ = aws_api.get_cluster_vpc_details(account)
             cluster["vpc_id"] = vpc_id
-            egress_ips = aws_api.get_cluster_nat_gateways_egress_ips(account)
+            egress_ips = aws_api.get_cluster_nat_gateways_egress_ips(account, vpc_id)
             cluster["egress_ips"] = ", ".join(sorted(egress_ips))
 
     # TODO(mafriedm): fix this
@@ -916,6 +1011,160 @@ def clusters_network(ctx, name):
     print_output(ctx.obj["options"], clusters, columns)
 
 
+@get.command()
+@click.pass_context
+def network_reservations(ctx) -> None:
+    from reconcile.typed_queries.reserved_networks import get_networks
+
+    columns = [
+        "name",
+        "network Address",
+        "parent Network",
+        "Account Name",
+        "Account UID",
+        "Console Login URL",
+    ]
+    network_table = []
+
+    def md_link(url) -> str:
+        if ctx.obj["options"]["output"] == "md":
+            return f"[{url}]({url})"
+        else:
+            return url
+
+    for network in get_networks():
+        parentAddress = "none"
+        if network.parent_network:
+            parentAddress = network.parent_network.network_address
+        if network.in_use_by and network.in_use_by.vpc:
+            network_table.append({
+                "name": network.name,
+                "network Address": network.network_address,
+                "parent Network": parentAddress,
+                "Account Name": network.in_use_by.vpc.account.name,
+                "Account UID": network.in_use_by.vpc.account.uid,
+                "Console Login URL": md_link(network.in_use_by.vpc.account.console_url),
+            })
+        else:
+            network_table.append({
+                "name": network.name,
+                "network Address": network.network_address,
+                "parent Network": parentAddress,
+                "Account Name": "Unclaimed network",
+                "Account UID": "Unclaimed network",
+                "Console Login URL": "Unclaimed network",
+            })
+    print_output(ctx.obj["options"], network_table, columns)
+
+
+@get.command()
+@click.option(
+    "--for-cluster",
+    help="If it is for getting cidr block for a cluster.",
+    type=bool,
+    default=False,
+)
+@click.option(
+    "--mask",
+    help="Mask for the latest available CIDR block for AWS resources. A decimal number between 1~32.",
+    type=int,
+    default=24,
+)
+@click.pass_context
+def cidr_blocks(ctx, for_cluster: int, mask: int) -> None:
+    import ipaddress
+
+    from reconcile.typed_queries.aws_vpcs import get_aws_vpcs
+
+    columns = ["type", "name", "account", "cidr", "from", "to", "hosts", "overlaps"]
+
+    clusters = [c for c in queries.get_clusters() if c.get("network")]
+    cidrs = [
+        {
+            "type": "cluster",
+            "name": c["name"],
+            "account": ((c.get("spec") or {}).get("account") or {}).get("name"),
+            "cidr": c["network"]["vpc"],
+            "from": str(ipaddress.ip_network(c["network"]["vpc"])[0]),
+            "to": str(ipaddress.ip_network(c["network"]["vpc"])[-1]),
+            "hosts": str(ipaddress.ip_network(c["network"]["vpc"]).num_addresses),
+            "description": c.get("description"),
+        }
+        for c in clusters
+    ]
+
+    tgw_cidrs = [
+        {
+            "type": "account-tgw",
+            "name": connection["account"]["name"],
+            "account": connection["account"]["name"],
+            "cidr": cidr,
+            "from": str(ipaddress.ip_network(cidr)[0]),
+            "to": str(ipaddress.ip_network(cidr)[-1]),
+            "hosts": str(ipaddress.ip_network(cidr).num_addresses),
+            "description": f'CIDR {cidr} routed through account {connection["account"]["name"]} transit gateways',
+        }
+        for c in clusters
+        for connection in (c["peering"] or {}).get("connections") or []
+        if connection["provider"] == "account-tgw"
+        for cidr in [connection["cidrBlock"]] + (connection["cidrBlocks"] or [])
+        if cidr is not None
+    ]
+    # removing dupes using a set of tuple (since dicts are not hashable)
+    unique_tgw_cidrs = [dict(t) for t in {tuple(d.items()) for d in tgw_cidrs}]
+    cidrs.extend(unique_tgw_cidrs)
+
+    vpcs = get_aws_vpcs()
+    cidrs.extend(
+        {
+            "type": "vpc",
+            "name": vpc.name,
+            "account": vpc.account.name,
+            "cidr": vpc.cidr_block,
+            "from": str(ipaddress.ip_network(vpc.cidr_block)[0]),
+            "to": str(ipaddress.ip_network(vpc.cidr_block)[-1]),
+            "hosts": str(ipaddress.ip_network(vpc.cidr_block).num_addresses),
+            "description": vpc.description,
+        }
+        for vpc in vpcs
+    )
+
+    for index, cidr in enumerate(cidrs):
+        network = ipaddress.ip_network(cidr["cidr"])
+        overlaps = [
+            f"{c['type']}/{c['name']}"
+            for i, c in enumerate(cidrs)
+            if i != index and network.overlaps(ipaddress.ip_network(c["cidr"]))
+        ]
+        cidr["overlaps"] = ", ".join(overlaps)
+
+    cidrs.sort(key=lambda item: ipaddress.ip_network(item["cidr"]))
+
+    if for_cluster:
+        latest_cluster_cidr = next(
+            (item for item in reversed(cidrs) if item["type"] == "cluster"),
+            None,
+        )
+
+        if not latest_cluster_cidr:
+            print("ERROR: Unable to find any existing cluster CIDR block.")
+            sys.exit(1)
+
+        avail_addr = ipaddress.ip_address(latest_cluster_cidr["to"]) + 1
+
+        print(f"INFO: Latest available network address: {avail_addr!s}")
+        try:
+            result_cidr_block = str(ipaddress.ip_network((avail_addr, mask)))
+        except ValueError:
+            print(f"ERROR: Invalid CIDR Mask {mask} Provided.")
+            sys.exit(1)
+        print(f"INFO: You are reserving {2 ** (32 - mask)!s} network addresses.")
+        print(f"\nYou can use: {result_cidr_block!s}")
+    else:
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], cidrs, columns)
+
+
 def ocm_aws_infrastructure_access_switch_role_links_data() -> list[dict]:
     settings = queries.get_app_interface_settings()
     clusters = queries.get_clusters()
@@ -972,38 +1221,6 @@ def ocm_aws_infrastructure_access_switch_role_links(ctx):
         print_output(ctx.obj["options"], by_user[user], columns)
 
 
-@get.command()
-@click.pass_context
-def clusters_egress_ips(ctx):
-    settings = queries.get_app_interface_settings()
-    clusters = queries.get_clusters()
-    clusters = [
-        c
-        for c in clusters
-        if c.get("ocm") is not None
-        and c.get("awsInfrastructureManagementAccounts") is not None
-    ]
-    ocm_map = OCMMap(clusters=clusters, settings=settings)
-
-    results = []
-    for cluster in clusters:
-        cluster_name = cluster["name"]
-        management_account = tfvpc._get_default_management_account(cluster)
-        account = tfvpc._build_infrastructure_assume_role(
-            management_account, cluster, ocm_map.get(cluster_name)
-        )
-        if not account:
-            continue
-        account["resourcesDefaultRegion"] = management_account["resourcesDefaultRegion"]
-        with AWSApi(1, [account], settings=settings, init_users=False) as aws_api:
-            egress_ips = aws_api.get_cluster_nat_gateways_egress_ips(account)
-        item = {"cluster": cluster_name, "egress_ips": ", ".join(sorted(egress_ips))}
-        results.append(item)
-
-    columns = ["cluster", "egress_ips"]
-    print_output(ctx.obj["options"], results, columns)
-
-
 @get.command()
 @click.pass_context
 def clusters_aws_account_ids(ctx):
@@ -1014,6 +1231,13 @@ def clusters_aws_account_ids(ctx):
     results = []
     for cluster in clusters:
         cluster_name = cluster["name"]
+        if cluster["spec"].get("account"):
+            item = {
+                "cluster": cluster_name,
+                "aws_account_id": cluster["spec"]["account"]["uid"],
+            }
+            results.append(item)
+            continue
         ocm = ocm_map.get(cluster_name)
         aws_account_id = ocm.get_cluster_aws_account_id(cluster_name)
         item = {
@@ -1026,71 +1250,8 @@ def clusters_aws_account_ids(ctx):
     print_output(ctx.obj["options"], results, columns)
 
 
-@get.command()
-@click.pass_context
-def terraform_users_credentials(ctx) -> None:
-    credentials = []
-    state = init_state(integration="account-notifier")
-
-    skip_accounts, appsre_pgp_key, _ = tfu.get_reencrypt_settings()
-
-    if skip_accounts:
-        accounts, working_dirs, _, aws_api = tfu.setup(
-            False,
-            1,
-            skip_accounts,
-            account_name=None,
-            appsre_pgp_key=appsre_pgp_key,
-        )
-
-        tf = Terraform(
-            tfu.QONTRACT_INTEGRATION,
-            tfu.QONTRACT_INTEGRATION_VERSION,
-            tfu.QONTRACT_TF_PREFIX,
-            accounts,
-            working_dirs,
-            10,
-            aws_api,
-            init_users=True,
-        )
-        for account, output in tf.outputs.items():
-            if account in skip_accounts:
-                user_passwords = tf.format_output(output, tf.OUTPUT_TYPE_PASSWORDS)
-                console_urls = tf.format_output(output, tf.OUTPUT_TYPE_CONSOLEURLS)
-                for user_name, enc_password in user_passwords.items():
-                    item = {
-                        "account": account,
-                        "console_url": console_urls[account],
-                        "user_name": user_name,
-                        "encrypted_password": enc_password,
-                    }
-                    credentials.append(item)
-
-    secrets = state.ls()
-
-    def _get_secret(secret_key: str):
-        if secret_key.startswith("/output/"):
-            secret_data = state.get(secret_key[1:])
-            if secret_data["account"] not in skip_accounts:
-                return secret_data
-        return None
-
-    secret_result = threaded.run(
-        _get_secret,
-        secrets,
-        10,
-    )
-
-    for secret in secret_result:
-        if secret and secret["account"] not in skip_accounts:
-            credentials.append(secret)
-
-    columns = ["account", "console_url", "user_name", "encrypted_password"]
-    print_output(ctx.obj["options"], credentials, columns)
-
-
-@root.command()
-@click.argument("account_name")
+@root.command()
+@click.argument("account_name")
 @click.pass_context
 def user_credentials_migrate_output(ctx, account_name) -> None:
     accounts = queries.get_state_aws_accounts()
@@ -1155,8 +1316,9 @@ def aws_route53_zones(ctx):
 
 @get.command()
 @click.argument("cluster_name")
+@click.option("--cluster-admin/--no-cluster-admin", default=False)
 @click.pass_context
-def bot_login(ctx, cluster_name):
+def bot_login(ctx, cluster_name, cluster_admin):
     settings = queries.get_app_interface_settings()
     secret_reader = SecretReader(settings=settings)
     clusters = queries.get_clusters()
@@ -1167,7 +1329,10 @@ def bot_login(ctx, cluster_name):
 
     cluster = clusters[0]
     server = cluster["serverUrl"]
-    token = secret_reader.read(cluster["automationToken"])
+    automation_token_name = (
+        "clusterAdminAutomationToken" if cluster_admin else "automationToken"
+    )
+    token = secret_reader.read(cluster[automation_token_name])
     print(f"oc login --server {server} --token {token}")
 
 
@@ -1218,6 +1383,181 @@ def aws_creds(ctx, account_name):
     print(f"export AWS_SECRET_ACCESS_KEY={secret['aws_secret_access_key']}")
 
 
+@root.command()
+@click.option(
+    "--account-uid",
+    help="account UID of the account that owns the bucket",
+    required=True,
+)
+@click.option(
+    "--source-bucket",
+    help="aws bucket where the source statefile is stored",
+    required=True,
+)
+@click.option(
+    "--source-object-path",
+    help="path in the bucket where the statefile is stored",
+    required=True,
+)
+@click.option(
+    "--rename",
+    help="optionally rename the destination repo, otherwise keep the same name for the new location",
+)
+@click.option("--region", help="AWS region")
+@click.option(
+    "--force/--no-force",
+    help="Force the copy even if a statefile already exists at the destination",
+    default=False,
+)
+@click.pass_context
+def copy_tfstate(
+    ctx, source_bucket, source_object_path, account_uid, rename, region, force
+):
+    settings = queries.get_app_interface_settings()
+    secret_reader = SecretReader(settings=settings)
+    accounts = queries.get_aws_accounts(uid=account_uid, terraform_state=True)
+    if not accounts:
+        print(f"{account_uid} not found in App-Interface.")
+        sys.exit(1)
+    account = accounts[0]
+
+    # terraform repo stores its statefiles within a "folder" in AWS S3 which is defined in App-Interface
+    dest_folder = [
+        i
+        for i in account["terraformState"]["integrations"]
+        if i["integration"] == "terraform-repo"
+    ]
+    if not dest_folder:
+        logging.error(
+            "terraform-repo is missing a section in this account's '/dependencies/terraform-state-1.yml' file, please add one using the docs in https://gitlab.cee.redhat.com/service/app-interface/-/blob/master/docs/terraform-repo/getting-started.md?ref_type=heads#step-1-setup-aws-account and then try again"
+        )
+        return
+
+    dest_filename = ""
+    if rename:
+        dest_filename = rename.removesuffix(".tfstate")
+    else:
+        dest_filename = source_object_path.removesuffix(".tfstate")
+
+    dest_key = f"{dest_folder[0]['key']}/{dest_filename}-tf-repo.tfstate"
+    dest_bucket = account["terraformState"]["bucket"]
+
+    with AWSApi(1, accounts, settings, secret_reader) as aws:
+        session = aws.get_session(account["name"])
+        s3_client = aws.get_session_client(session, "s3", region)
+        copy_source = {
+            "Bucket": source_bucket,
+            "Key": source_object_path,
+        }
+
+        dest_pretty_path = f"s3://{dest_bucket}/{dest_key}"
+        # check if dest already exists
+        response = s3_client.list_objects_v2(
+            Bucket=dest_bucket, Prefix=dest_key, MaxKeys=1
+        )
+
+        if "Contents" in response:
+            if force:
+                logging.warning(
+                    f"Existing object at '{dest_pretty_path}' will be overwritten as --force is set"
+                )
+            else:
+                logging.error(
+                    f"Will not overwrite existing object at '{dest_pretty_path}'. Use --force to overwrite the destination object"
+                )
+                return
+
+        prompt_text = f"Are you sure you want to copy 's3://{source_bucket}/{source_object_path}' to '{dest_pretty_path}'?"
+        if click.confirm(prompt_text):
+            s3_client.copy(copy_source, dest_bucket, dest_key)
+            print(
+                textwrap.dedent(f"""
+                            Nicely done! Your tfstate file has been migrated. Now you can create a repo definition in App-Interface like so:
+
+                            ---
+                            $schema: /aws/terraform-repo-1.yml
+
+                            account:
+                                $ref: {account["path"]}
+
+                            name: {dest_filename}
+                            repository: <FILL_IN>
+                            projectPath: <FILL_IN>
+                            tfVersion: <FILL_IN>
+                            ref: <FILL_IN>""")
+            )
+
+
+@get.command(short_help='obtain "rosa create cluster" command by cluster name')
+@click.argument("cluster_name")
+@click.pass_context
+def rosa_create_cluster_command(ctx, cluster_name):
+    clusters = [c for c in get_clusters() if c.name == cluster_name]
+    try:
+        cluster = clusters[0]
+    except IndexError:
+        print(f"{cluster_name} not found.")
+        sys.exit(1)
+
+    if cluster.spec.product != OCM_PRODUCT_ROSA:
+        print("must be a rosa cluster.")
+        sys.exit(1)
+
+    settings = queries.get_app_interface_settings()
+    account = cluster.spec.account
+
+    if account.billing_account:
+        billing_account = account.billing_account.uid
+    else:
+        with AWSApi(
+            1, [account.dict(by_alias=True)], settings=settings, init_users=False
+        ) as aws_api:
+            billing_account = aws_api.get_organization_billing_account(account.name)
+
+    print(
+        " ".join([
+            "rosa create cluster",
+            f"--billing-account {billing_account}",
+            f"--cluster-name {cluster.name}",
+            "--sts",
+            ("--private" if cluster.spec.private else ""),
+            ("--hosted-cp" if cluster.spec.hypershift else ""),
+            (
+                "--private-link"
+                if cluster.spec.private and not cluster.spec.hypershift
+                else ""
+            ),
+            (
+                "--multi-az"
+                if cluster.spec.multi_az and not cluster.spec.hypershift
+                else ""
+            ),
+            f"--operator-roles-prefix {cluster.name}",
+            f"--oidc-config-id {cluster.spec.oidc_endpoint_url.split('/')[-1]}",
+            f"--subnet-ids {','.join(cluster.spec.subnet_ids)}",
+            f"--region {cluster.spec.region}",
+            f"--version {cluster.spec.initial_version}",
+            f"--machine-cidr {cluster.network.vpc}",
+            f"--service-cidr {cluster.network.service}",
+            f"--pod-cidr {cluster.network.pod}",
+            "--host-prefix 23",
+            "--replicas 3",
+            f"--compute-machine-type {cluster.machine_pools[0].instance_type}",
+            (
+                "--disable-workload-monitoring"
+                if cluster.spec.disable_user_workload_monitoring
+                else ""
+            ),
+            f"--channel-group {cluster.spec.channel}",
+            (
+                f"--properties provision_shard_id:{cluster.spec.provision_shard_id}"
+                if cluster.spec.provision_shard_id
+                else ""
+            ),
+        ])
+    )
+
+
 @get.command(
     short_help="obtain sshuttle command for "
     "connecting to private clusters via a jump host. "
@@ -1227,9 +1567,7 @@ def aws_creds(ctx, account_name):
 @click.argument("jumphost_hostname", required=False)
 @click.argument("cluster_name", required=False)
 @click.pass_context
-def sshuttle_command(
-    ctx, jumphost_hostname: Optional[str], cluster_name: Optional[str]
-):
+def sshuttle_command(ctx, jumphost_hostname: str | None, cluster_name: str | None):
     jumphosts_query_data = queries.get_jumphosts(hostname=jumphost_hostname)
     jumphosts = jumphosts_query_data.jumphosts or []
     for jh in jumphosts:
@@ -1350,6 +1688,190 @@ def aws_terraform_resources(ctx):
     print_output(ctx.obj["options"], results.values(), columns)
 
 
+def rds_attr(
+    attr: str, overrides: dict[str, str], defaults: dict[str, str]
+) -> str | None:
+    return overrides.get(attr) or defaults.get(attr)
+
+
+def region_from_az(az: str | None) -> str | None:
+    if not az:
+        return None
+    return az[:-1]
+
+
+def rds_region(
+    spec: ExternalResourceSpec,
+    overrides: dict[str, str],
+    defaults: dict[str, str],
+    accounts: dict[str, Any],
+) -> str | None:
+    return (
+        spec.resource.get("region")
+        or rds_attr("region", overrides, defaults)
+        or region_from_az(spec.resource.get("availability_zone"))
+        or region_from_az(rds_attr("availability_zone", overrides, defaults))
+        or accounts[spec.provisioner_name].get("resourcesDefaultRegion")
+    )
+
+
+@get.command
+@click.pass_context
+def rds(ctx):
+    namespaces = tfr.get_namespaces()
+    accounts = {a["name"]: a for a in queries.get_aws_accounts()}
+    results = []
+    for namespace in namespaces:
+        specs = [
+            s
+            for s in get_external_resource_specs(
+                namespace.dict(by_alias=True), provision_provider=PROVIDER_AWS
+            )
+            if s.provider == "rds"
+        ]
+        for spec in specs:
+            defaults = yaml.safe_load(
+                gql.get_resource(spec.resource["defaults"])["content"]
+            )
+            overrides = json.loads(spec.resource.get("overrides") or "{}")
+            item = {
+                "identifier": spec.identifier,
+                "account": spec.provisioner_name,
+                "account_uid": accounts[spec.provisioner_name]["uid"],
+                "region": rds_region(spec, overrides, defaults, accounts),
+                "engine": rds_attr("engine", overrides, defaults),
+                "engine_version": rds_attr("engine_version", overrides, defaults),
+                "instance_class": rds_attr("instance_class", overrides, defaults),
+                "storage_type": rds_attr("storage_type", overrides, defaults),
+                "ca_cert_identifier": rds_attr(
+                    "ca_cert_identifier", overrides, defaults
+                ),
+            }
+            results.append(item)
+
+    if ctx.obj["options"]["output"] == "md":
+        json_table = {
+            "filter": True,
+            "fields": [
+                {"key": "identifier", "sortable": True},
+                {"key": "account", "sortable": True},
+                {"key": "account_uid", "sortable": True},
+                {"key": "region", "sortable": True},
+                {"key": "engine", "sortable": True},
+                {"key": "engine_version", "sortable": True},
+                {"key": "instance_class", "sortable": True},
+                {"key": "storage_type", "sortable": True},
+                {"key": "ca_cert_identifier", "sortable": True},
+            ],
+            "items": results,
+        }
+
+        print(
+            f"""
+You can view the source of this Markdown to extract the JSON data.
+
+{len(results)} RDS instances found.
+
+```json:table
+{json.dumps(json_table)}
+```
+            """
+        )
+    else:
+        columns = [
+            "identifier",
+            "account",
+            "account_uid",
+            "region",
+            "engine",
+            "engine_version",
+            "instance_class",
+            "storage_type",
+            "ca_cert_identifier",
+        ]
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], results, columns)
+
+
+@get.command
+@click.pass_context
+def rds_recommendations(ctx):
+    IGNORED_STATUSES = ("resolved",)
+    IGNORED_SEVERITIES = ("informational",)
+
+    settings = queries.get_app_interface_settings()
+
+    # Only check AWS accounts for which we have RDS resources defined
+    targetted_accounts = []
+    namespaces = queries.get_namespaces()
+    for namespace_info in namespaces:
+        if not managed_external_resources(namespace_info):
+            continue
+        for spec in get_external_resource_specs(namespace_info):
+            if spec.provider == "rds":
+                targetted_accounts.append(spec.provisioner_name)
+
+    accounts = [
+        a for a in queries.get_aws_accounts() if a["name"] in targetted_accounts
+    ]
+    accounts.sort(key=lambda a: a["name"])
+
+    columns = [
+        # 'RecommendationId',
+        # 'TypeId',
+        # 'ResourceArn',
+        "ResourceName",  # Non-AWS field
+        "Severity",
+        "Category",
+        "Impact",
+        "Status",
+        "Detection",
+        "Recommendation",
+        "Description",
+        # 'Source',
+        # 'TypeDetection',
+        # 'TypeRecommendation',
+        # 'AdditionalInfo'
+    ]
+
+    ctx.obj["options"]["sort"] = False
+
+    print("[TOC]")
+    for account in accounts:
+        account_name = account.get("name")
+        account_deployment_regions = account.get("supportedDeploymentRegions")
+        for region in account_deployment_regions or []:
+            with AWSApi(1, [account], settings=settings, init_users=False) as aws:
+                try:
+                    data = aws.describe_rds_recommendations(account_name, region)
+                    recommendations = data.get("DBRecommendations", [])
+                except Exception as e:
+                    logging.error(f"Error describing RDS recommendations: {e}")
+                    continue
+
+                # Add field ResourceName infered from ResourceArn
+                recommendations = [
+                    {**rec, "ResourceName": rec["ResourceArn"].split(":")[-1]}
+                    for rec in recommendations
+                    if rec.get("Status") not in IGNORED_STATUSES
+                    and rec.get("Severity") not in IGNORED_SEVERITIES
+                ]
+                # The Description field has \n that are causing issues with the markdown table
+                recommendations = [
+                    {**rec, "Description": rec["Description"].replace("\n", " ")}
+                    for rec in recommendations
+                ]
+                # If we have no recommendations to show, skip
+                if not recommendations:
+                    continue
+                # Sort by ResourceName
+                recommendations.sort(key=lambda r: r["ResourceName"])
+
+                print(f"# {account_name} - {region}")
+                print("Note: Severity informational is not shown.")
+                print_output(ctx.obj["options"], recommendations, columns)
+
+
 @get.command()
 @click.pass_context
 def products(ctx):
@@ -1434,17 +1956,8 @@ def roles(ctx, org_username):
 
     user = users[0]
 
-    roles = []
-
-    def add(d):
-        for i, r in enumerate(roles):
-            if all(d[k] == r[k] for k in ("type", "name", "resource")):
-                roles.insert(
-                    i + 1, {"type": "", "name": "", "resource": "", "ref": d["ref"]}
-                )
-                return
-
-        roles.append(d)
+    # type, name, resource, [ref]
+    roles: dict[(str, str, str), set] = defaultdict(set)
 
     for role in user["roles"]:
         role_name = role["path"]
@@ -1461,63 +1974,38 @@ def roles(ctx, org_username):
             if "team" in p:
                 r_name += "/" + p["team"]
 
-            add(
-                {
-                    "type": "permission",
-                    "name": p["name"],
-                    "resource": r_name,
-                    "ref": role_name,
-                }
-            )
+            roles["permission", p["name"], r_name].add(role_name)
 
         for aws in role.get("aws_groups") or []:
             for policy in aws["policies"]:
-                add(
-                    {
-                        "type": "aws",
-                        "name": policy,
-                        "resource": aws["account"]["name"],
-                        "ref": aws["path"],
-                    }
-                )
+                roles["aws", policy, aws["account"]["name"]].add(aws["path"])
 
         for a in role.get("access") or []:
             if a["cluster"]:
                 cluster_name = a["cluster"]["name"]
-                add(
-                    {
-                        "type": "cluster",
-                        "name": a["clusterRole"],
-                        "resource": cluster_name,
-                        "ref": role_name,
-                    }
-                )
+                roles["cluster", a["clusterRole"], cluster_name].add(role_name)
             elif a["namespace"]:
                 ns_name = a["namespace"]["name"]
-                add(
-                    {
-                        "type": "namespace",
-                        "name": a["role"],
-                        "resource": ns_name,
-                        "ref": role_name,
-                    }
-                )
+                roles["namespace", a["role"], ns_name].add(role_name)
 
         for s in role.get("self_service") or []:
             for d in s.get("datafiles") or []:
                 name = d.get("name")
                 if name:
-                    add(
-                        {
-                            "type": "saas_file",
-                            "name": "owner",
-                            "resource": name,
-                            "ref": role_name,
-                        }
-                    )
+                    roles["saas_file", "owner", name].add(role_name)
 
     columns = ["type", "name", "resource", "ref"]
-    print_output(ctx.obj["options"], roles, columns)
+    rows = [
+        {
+            "type": k[0],
+            "name": k[1],
+            "resource": k[2],
+            "ref": ref,
+        }
+        for k, v in roles.items()
+        for ref in v
+    ]
+    print_output(ctx.obj["options"], rows, columns)
 
 
 @get.command()
@@ -1564,13 +2052,11 @@ def quay_mirrors(ctx):
                 url = item["mirror"]["url"]
                 public = item["public"]
 
-                mirrors.append(
-                    {
-                        "repo": f"quay.io/{org_name}/{name}",
-                        "public": public,
-                        "upstream": url,
-                    }
-                )
+                mirrors.append({
+                    "repo": f"quay.io/{org_name}/{name}",
+                    "public": public,
+                    "upstream": url,
+                })
 
     columns = ["repo", "upstream", "public"]
     print_output(ctx.obj["options"], mirrors, columns)
@@ -1689,7 +2175,7 @@ def app_interface_merge_queue(ctx):
             ).total_seconds()
             / 60,
             "approved_by": mr["approved_by"],
-            "labels": ", ".join(mr["labels"]),
+            "labels": ", ".join(mr["mr"].labels),
         }
         merge_queue_data.append(item)
 
@@ -1727,14 +2213,14 @@ def app_interface_review_queue(ctx) -> None:
 
         queue_data = []
         for mr in merge_requests:
-            if mr.work_in_progress:
+            if mr.draft:
                 continue
             if len(mr.commits()) == 0:
                 continue
-            if mr.merge_status in [
+            if mr.merge_status in {
                 MRStatus.CANNOT_BE_MERGED,
                 MRStatus.CANNOT_BE_MERGED_RECHECK,
-            ]:
+            }:
                 continue
 
             labels = mr.attributes.get("labels")
@@ -1744,10 +2230,14 @@ def app_interface_review_queue(ctx) -> None:
                 continue
             if SAAS_FILE_UPDATE in labels:
                 continue
-            if SELF_SERVICEABLE in labels:
+            if (
+                SELF_SERVICEABLE in labels
+                and SHOW_SELF_SERVICEABLE_IN_REVIEW_QUEUE not in labels
+                and AVS not in labels
+            ):
                 continue
 
-            pipelines = mr.pipelines()
+            pipelines = gl.get_merge_request_pipelines(mr)
             if not pipelines:
                 continue
             running_pipelines = [p for p in pipelines if p["status"] == "running"]
@@ -1799,7 +2289,10 @@ def app_interface_review_queue(ctx) -> None:
 
     queue_data.sort(key=itemgetter("updated_at"))
     ctx.obj["options"]["sort"] = False  # do not sort
-    print_output(ctx.obj["options"], queue_data, columns)
+    text = print_output(ctx.obj["options"], queue_data, columns)
+    if text:
+        slack = slackapi_from_queries("app-interface-review-queue")
+        slack.chat_post_message("```\n" + text + "\n```")
 
 
 @get.command()
@@ -1819,7 +2312,7 @@ def app_interface_open_selfserviceable_mr_queue(ctx):
     ]
     queue_data = []
     for mr in merge_requests:
-        if mr.work_in_progress:
+        if mr.draft:
             continue
         if len(mr.commits()) == 0:
             continue
@@ -1841,7 +2334,7 @@ def app_interface_open_selfserviceable_mr_queue(ctx):
             continue
 
         # skip MRs where the pipeline is still running or where it failed
-        pipelines = mr.pipelines()
+        pipelines = gl.get_merge_request_pipelines(mr)
         if not pipelines:
             continue
         running_pipelines = [p for p in pipelines if p["status"] == "running"]
@@ -1879,14 +2372,12 @@ def change_types(ctx) -> None:
             usage_statistics[ss.change_type.name] += nr_files
     data = []
     for ct in change_types:
-        data.append(
-            {
-                "name": ct.name,
-                "description": ct.description,
-                "applicable to": f"{ct.context_type.value} {ct.context_schema or '' }",
-                "# usages": usage_statistics[ct.name],
-            }
-        )
+        data.append({
+            "name": ct.name,
+            "description": ct.description,
+            "applicable to": f"{ct.context_type.value} {ct.context_schema or ''}",
+            "# usages": usage_statistics[ct.name],
+        })
     columns = ["name", "description", "applicable to", "# usages"]
     print_output(ctx.obj["options"], data, columns)
 
@@ -1897,7 +2388,11 @@ def app_interface_merge_history(ctx):
     settings = queries.get_app_interface_settings()
     instance = queries.get_gitlab_instance()
     gl = GitLabApi(instance, project_url=settings["repoUrl"], settings=settings)
-    merge_requests = gl.project.mergerequests.list(state=MRState.MERGED, per_page=100)
+    merge_requests = gl.project.mergerequests.list(
+        state=MRState.MERGED,
+        per_page=100,
+        get_all=False,
+    )
 
     columns = [
         "id",
@@ -2043,69 +2538,637 @@ def selectorsyncset_managed_hypershift_resources(ctx, use_jump_host):
     print_output(ctx.obj["options"], data, columns)
 
 
-@root.group()
-@output
+@get.command()
+@click.option(
+    "--aws-access-key-id",
+    help="AWS access key id",
+    default=os.environ.get("QONTRACT_CLI_EC2_JENKINS_WORKER_AWS_ACCESS_KEY_ID", None),
+)
+@click.option(
+    "--aws-secret-access-key",
+    help="AWS secret access key",
+    default=os.environ.get(
+        "QONTRACT_CLI_EC2_JENKINS_WORKER_AWS_SECRET_ACCESS_KEY", None
+    ),
+)
+@click.option(
+    "--aws-region",
+    help="AWS region",
+    default=os.environ.get("QONTRACT_CLI_EC2_JENKINS_WORKER_AWS_REGION", "us-east-1"),
+)
 @click.pass_context
-def set(ctx, output):
-    ctx.obj["output"] = output
+def ec2_jenkins_workers(ctx, aws_access_key_id, aws_secret_access_key, aws_region):
+    """Prints a list of jenkins workers and their status."""
+    if not aws_access_key_id or not aws_secret_access_key:
+        raise click.ClickException(
+            "AWS credentials not provided. Either set them in the environment "
+            "QONTRACT_CLI_EC2_JENKINS_WORKER_AWS_ACCESS_KEY_ID "
+            "and QONTRACT_CLI_EC2_JENKINS_WORKER_AWS_SECRET_ACCESS_KEY "
+            "or pass them as arguments."
+        )
 
+    boto3.setup_default_session(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        region_name=aws_region,
+    )
+    client = boto3.client("autoscaling")
+    ec2 = boto3.resource("ec2")
+    results = []
+    now = datetime.now(UTC)
+    DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+    columns = [
+        "type",
+        "id",
+        "IP",
+        "instance type",
+        "launch time (utc)",
+        "OS",
+        "AMI",
+    ]
 
-@set.command()
-@click.argument("workspace")
-@click.argument("usergroup")
-@click.argument("username")
-@click.pass_context
-def slack_usergroup(ctx, workspace, usergroup, username):
-    """Update users in a slack usergroup.
-    Use an org_username as the username.
-    To empty a slack usergroup, pass '' (empty string) as the username.
-    """
-    settings = queries.get_app_interface_settings()
-    slack = slackapi_from_queries("qontract-cli")
-    ugid = slack.get_usergroup_id(usergroup)
-    if username:
-        mail_address = settings["smtp"]["mailAddress"]
-        users = [slack.get_user_id_by_name(username, mail_address)]
-    else:
-        users = [slack.get_random_deleted_user()]
-    slack.update_usergroup_users(ugid, users)
+    auto_scaling_groups = client.describe_auto_scaling_groups()["AutoScalingGroups"]
+    for a in auto_scaling_groups:
+        for i in a["Instances"]:
+            lifecycle_state = i["LifecycleState"]
+            if lifecycle_state != "InService":
+                logging.info(
+                    f"instance is in lifecycle state {lifecycle_state} - ignoring instance"
+                )
+                continue
+            instance = ec2.Instance(i["InstanceId"])
+            state = instance.state["Name"]
+            if state != "running":
+                continue
+            os = ""
+            url = ""
+            for t in instance.tags:
+                if t.get("Key") == "os":
+                    os = t.get("Value")
+                if t.get("Key") == "jenkins_controller":
+                    url = f"https://{t.get('Value').replace('-', '.')}.devshift.net/computer/{instance.id}"
+            image = ec2.Image(instance.image_id)
+            commit_url = ""
+            for t in image.tags:
+                if t.get("Key") == "infra_commit":
+                    commit_url = f"https://gitlab.cee.redhat.com/app-sre/infra/-/tree/{t.get('Value')}"
+            launch_emoji = "💫"
+            launch_hours = (now - instance.launch_time).total_seconds() / 3600
+            if launch_hours > 24:
+                launch_emoji = "⏰"
+            item = {
+                "type": a["AutoScalingGroupName"],
+                "id": f"[{instance.id}]({url})",
+                "IP": instance.private_ip_address,
+                "instance type": instance.instance_type,
+                "launch time (utc)": f"{instance.launch_time.strftime(DATE_FORMAT)} {launch_emoji}",
+                "OS": os,
+                "AMI": f"[{image.name}]({commit_url})",
+            }
+            results.append(item)
 
+    print_output(ctx.obj["options"], results, columns)
 
-@set.command()
-@click.argument("org_name")
-@click.argument("cluster_name")
+
+@get.command()
+@click.argument("status-board-instance")
 @click.pass_context
-def cluster_admin(ctx, org_name, cluster_name):
-    settings = queries.get_app_interface_settings()
-    ocms = [
-        o for o in queries.get_openshift_cluster_managers() if o["name"] == org_name
+def slo_document_services(ctx, status_board_instance):
+    """Print SLO Documents Services"""
+    columns = [
+        "slo_doc_name",
+        "product",
+        "app",
+        "slo",
+        "sli_type",
+        "sli_specification",
+        "slo_details",
+        "target",
+        "target_unit",
+        "window",
+        "statusBoardEnabled",
     ]
-    ocm_map = OCMMap(ocms=ocms, settings=settings)
-    ocm = ocm_map[org_name]
-    enabled = ocm.is_cluster_admin_enabled(cluster_name)
-    if not enabled:
-        ocm.enable_cluster_admin(cluster_name)
 
+    try:
+        [sb] = [sb for sb in get_status_board() if sb.name == status_board_instance]
+    except ValueError:
+        print(f"Status-board instance '{status_board_instance}' not found.")
+        sys.exit(1)
 
-@root.group()
-@environ(["APP_INTERFACE_STATE_BUCKET", "APP_INTERFACE_STATE_BUCKET_ACCOUNT"])
-@click.pass_context
-def state(ctx):
-    pass
+    desired_product_apps: dict[str, set[str]] = (
+        StatusBoardExporterIntegration.get_product_apps(sb)
+    )
 
+    slodocs = []
+    for slodoc in get_slo_documents():
+        products = [ns.namespace.environment.product.name for ns in slodoc.namespaces]
+        for slo in slodoc.slos:
+            for product in products:
+                if slodoc.app.parent_app:
+                    app = f"{slodoc.app.parent_app.name}-{slodoc.app.name}"
+                else:
+                    app = slodoc.app.name
+
+                # Skip if the (product, app) is not being generated by the status-board inventory
+                if (
+                    product not in desired_product_apps
+                    or app not in desired_product_apps[product]
+                ):
+                    continue
 
-@state.command()
-@click.argument("integration", default="")
+                item = {
+                    "slo_doc_name": slodoc.name,
+                    "product": product,
+                    "app": app,
+                    "slo": slo.name,
+                    "sli_type": slo.sli_type,
+                    "sli_specification": slo.sli_specification,
+                    "slo_details": slo.slo_details,
+                    "target": slo.slo_target,
+                    "target_unit": slo.slo_target_unit,
+                    "window": slo.slo_parameters.window,
+                    "statusBoardService": f"{product}/{slodoc.app.name}/{slo.name}",
+                    "statusBoardEnabled": "statusBoard" in slodoc.labels,
+                }
+                slodocs.append(item)
+
+    print_output(ctx.obj["options"], slodocs, columns)
+
+
+@get.command()
+@click.argument("file_path")
 @click.pass_context
-def ls(ctx, integration):
-    state = init_state(integration=integration)
-    keys = state.ls()
-    # if integration in not defined the 2th token will be the integration name
-    key_index = 1 if integration else 2
-    table_content = [
-        {
-            "integration": integration or k.split("/")[1],
-            "key": "/".join(k.split("/")[key_index:]),
+def alerts(ctx, file_path):
+    BIG_NUMBER = 10
+
+    def sort_by_threshold(item: dict[str, str]) -> int:
+        threshold = item["threshold"]
+        if not threshold:
+            return BIG_NUMBER * 60 * 24
+        value = int(threshold[:-1])
+        unit = threshold[-1]
+        match unit:
+            case "m":
+                return value
+            case "h":
+                return value * 60
+            case "d":
+                return value * 60 * 24
+            case _:
+                return BIG_NUMBER * 60 * 24
+
+    def sort_by_severity(item: dict[str, str]) -> int:
+        match item["severity"].lower():
+            case "critical":
+                return 0
+            case "warning":
+                return 1
+            case "info":
+                return 2
+            case _:
+                return BIG_NUMBER
+
+    with open(file_path, encoding="locale") as f:
+        content = json.loads(f.read())
+
+    columns = [
+        "name",
+        "summary",
+        "severity",
+        "threshold",
+        "description",
+    ]
+    data = []
+    prometheus_rules = content["items"]
+    for prom_rule in prometheus_rules:
+        groups = prom_rule["spec"]["groups"]
+        for group in groups:
+            rules = group["rules"]
+            for rule in rules:
+                name = rule.get("alert")
+                summary = rule.get("annotations", {}).get("summary")
+                message = rule.get("annotations", {}).get("message")
+                severity = rule.get("labels", {}).get("severity")
+                description = rule.get("annotations", {}).get("description")
+                threshold = rule.get("for")
+                if name:
+                    data.append({
+                        "name": name,
+                        "summary": "`" + (summary or message).replace("\n", " ") + "`"
+                        if summary or message
+                        else "",
+                        "severity": severity,
+                        "threshold": threshold,
+                        "description": "`" + description.replace("\n", " ") + "`"
+                        if description
+                        else "",
+                    })
+    ctx.obj["options"]["sort"] = False
+    data = sorted(data, key=sort_by_threshold)
+    data = sorted(data, key=sort_by_severity)
+    print_output(ctx.obj["options"], data, columns)
+
+
+@get.command()
+@click.pass_context
+def aws_cost_report(ctx):
+    command = AwsCostReportCommand.create()
+    print(command.execute())
+
+
+@get.command()
+@click.pass_context
+def openshift_cost_report(ctx):
+    command = OpenShiftCostReportCommand.create()
+    print(command.execute())
+
+
+@get.command()
+@click.pass_context
+def openshift_cost_optimization_report(ctx):
+    command = OpenShiftCostOptimizationReportCommand.create()
+    print(command.execute())
+
+
+@get.command()
+@click.pass_context
+def osd_component_versions(ctx):
+    osd_environments = [
+        e["name"] for e in queries.get_environments() if e["product"]["name"] == "OSDv4"
+    ]
+    data = []
+    saas_files = get_saas_files()
+    for sf in saas_files:
+        for rt in sf.resource_templates:
+            for t in rt.targets:
+                if t.namespace.environment.name not in osd_environments:
+                    continue
+                item = {
+                    "environment": t.namespace.environment.name,
+                    "namespace": t.namespace.name,
+                    "cluster": t.namespace.cluster.name,
+                    "app": sf.app.name,
+                    "saas_file": sf.name,
+                    "resource_template": rt.name,
+                    "ref": f"[{t.ref}]({rt.url}/blob/{t.ref}{rt.path})",
+                }
+                data.append(item)
+
+    columns = [
+        "environment",
+        "namespace",
+        "cluster",
+        "app",
+        "saas_file",
+        "resource_template",
+        "ref",
+    ]
+    print_output(ctx.obj["options"], data, columns)
+
+
+@get.command()
+@click.pass_context
+def maintenances(ctx):
+    now = datetime.now(UTC)
+    maintenances = maintenances_gql.query(gql.get_api().query).maintenances or []
+    data = [
+        {
+            **m.dict(),
+            "services": ", ".join(a.name for a in m.affected_services),
+        }
+        for m in maintenances
+        if datetime.fromisoformat(m.scheduled_start) > now
+    ]
+    columns = [
+        "name",
+        "scheduled_start",
+        "scheduled_end",
+        "services",
+    ]
+    print_output(ctx.obj["options"], data, columns)
+
+
+class MigrationStatusCount:
+    def __init__(self, app: str) -> None:
+        self.app = app
+        self._source = 0
+        self._target = 0
+
+    def inc(self, source_or_target: str) -> None:
+        match source_or_target:
+            case "source":
+                self._source += 1
+            case "target":
+                self._target += 1
+            case _:
+                raise ValueError("hcp migration label must be source or target")
+
+    @property
+    def classic(self) -> int:
+        return self._source
+
+    @property
+    def hcp(self) -> int:
+        return self._target
+
+    @property
+    def total(self) -> int:
+        return self.classic + self.hcp
+
+    @property
+    def progress(self) -> float:
+        return round(self.hcp / self.total * 100, 0)
+
+    @property
+    def item(self) -> dict[str, Any]:
+        return {
+            "app": self.app,
+            "classic": self.classic or "0",
+            "hcp": self.hcp or "0",
+            "progress": self.progress or "0",
+        }
+
+
+@get.command()
+@click.pass_context
+def hcp_migration_status(ctx):
+    counts: dict[str, MigrationStatusCount] = {}
+    total_count = MigrationStatusCount("total")
+    saas_files = get_saas_files()
+    for sf in saas_files:
+        if sf.publish_job_logs:
+            # ignore post deployment test saas files
+            continue
+        for rt in sf.resource_templates:
+            if rt.provider == "directory" or "dashboard" in rt.name:
+                # ignore grafana dashboards
+                continue
+            for t in rt.targets:
+                if t.namespace.name.startswith("openshift-"):
+                    # ignore openshift namespaces
+                    continue
+                if t.namespace.path.startswith("/openshift/"):
+                    # ignore per-cluster namespaces
+                    continue
+                if t.delete:
+                    continue
+                if hcp_migration := t.namespace.cluster.labels.get("hcp_migration"):
+                    app = sf.app.parent_app.name if sf.app.parent_app else sf.app.name
+                    counts.setdefault(app, MigrationStatusCount(app))
+                    counts[app].inc(hcp_migration)
+                    total_count.inc(hcp_migration)
+
+    data = [c.item for c in counts.values()]
+    print(
+        f"SUMMARY: {total_count.hcp} / {total_count.total} COMPLETED ({total_count.progress}%)"
+    )
+    columns = ["app", "classic", "hcp", "progress"]
+    print_output(ctx.obj["options"], data, columns)
+
+
+@get.command()
+@click.pass_context
+def systems_and_tools(ctx):
+    print(
+        f"This report is obtained from app-interface Graphql endpoint available at: {config.get_config()['graphql']['server']}"
+    )
+    inventory = get_systems_and_tools_inventory()
+    print_output(ctx.obj["options"], inventory.data, inventory.columns)
+
+
+@get.command(short_help="get integration logs")
+@click.argument("integration_name")
+@click.option(
+    "--environment_name", default="production", help="environment to get logs from"
+)
+@click.pass_context
+def logs(ctx, integration_name: str, environment_name: str):
+    integrations = [
+        i
+        for i in integrations_gql.query(query_func=gql.get_api().query).integrations
+        or []
+        if i.name == integration_name
+    ]
+    if not integrations:
+        print("integration not found")
+        return
+    integration = integrations[0]
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    managed = integration.managed
+    if not managed:
+        print("integration is not managed")
+        return
+    namespaces = [
+        m.namespace
+        for m in managed
+        if m.namespace.cluster.labels
+        and m.namespace.cluster.labels.get("environment") == environment_name
+    ]
+    if not namespaces:
+        print(f"no managed {environment_name} namespace found")
+        return
+    namespace = namespaces[0]
+    cluster = namespaces[0].cluster
+    if not cluster.automation_token:
+        print("cluster automation token not found")
+        return
+    token = secret_reader.read_secret(cluster.automation_token)
+
+    command = f"oc --server {cluster.server_url} --token {token} --namespace {namespace.name} logs -c int -l app=qontract-reconcile-{integration.name}"
+    print(command)
+
+
+@get.command
+@click.pass_context
+def jenkins_jobs(ctx):
+    jenkins_configs = queries.get_jenkins_configs()
+
+    # stats dicts
+    apps = {}
+    totals = {"rhel8": 0, "other": 0}
+
+    for jc in jenkins_configs:
+        app_name = jc["app"]["name"]
+
+        if app_name not in apps:
+            apps[app_name] = {"rhel8": 0, "other": 0}
+
+        config = json.loads(jc["config"]) if jc["config"] else []
+        for c in config:
+            if "project" not in c:
+                continue
+
+            project = c["project"]
+            root_node = project.get("node") or ""
+            if "jobs" not in project:
+                continue
+
+            for pj in project["jobs"]:
+                for job in pj.values():
+                    node = job.get("node", root_node)
+                    if node in {"rhel8", "rhel8-app-interface"}:
+                        apps[app_name]["rhel8"] += 1
+                        totals["rhel8"] += 1
+                    else:
+                        apps[app_name]["other"] += 1
+                        totals["other"] += 1
+
+    results = [
+        {"app": app} | stats
+        for app, stats in sorted(apps.items(), key=lambda i: i[0].lower())
+        if not (stats["other"] == 0 and stats["rhel8"] == 0)
+    ]
+    results.append({"app": "TOTALS"} | totals)
+
+    if ctx.obj["options"]["output"] == "md":
+        json_table = {
+            "filter": True,
+            "fields": [
+                {"key": "app"},
+                {"key": "other"},
+                {"key": "rhel8"},
+            ],
+            "items": results,
+        }
+
+        print(
+            f"""
+You can view the source of this Markdown to extract the JSON data.
+
+{len(results)} apps with Jenkins jobs
+
+```json:table
+{json.dumps(json_table)}
+```
+            """
+        )
+    else:
+        columns = ["app", "other", "rhel8"]
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], results, columns)
+
+
+@get.command
+@click.pass_context
+def container_image_details(ctx):
+    apps = get_apps_quay_repos_escalation_policies()
+    data: list[dict[str, str]] = []
+    for app in apps:
+        app_name = f"{app.parent_app.name}/{app.name}" if app.parent_app else app.name
+        ep_channels = app.escalation_policy.channels
+        email = ep_channels.email
+        slack = ep_channels.slack_user_group[0].handle
+        for org_items in app.quay_repos or []:
+            org_name = org_items.org.name
+            for repo in org_items.items or []:
+                if repo.mirror:
+                    continue
+                repository = f"quay.io/{org_name}/{repo.name}"
+                item = {
+                    "app": app_name,
+                    "repository": repository,
+                    "email": email,
+                    "slack": slack,
+                }
+                data.append(item)
+    columns = ["app", "repository", "email", "slack"]
+    print_output(ctx.obj["options"], data, columns)
+
+
+@get.command
+@click.pass_context
+def change_log_tracking(ctx):
+    repo_url = get_app_interface_repo_url()
+    change_types = fetch_change_type_processors(gql.get_api(), NoOpFileDiffResolver())
+    state = init_state(integration=cl.QONTRACT_INTEGRATION)
+    change_log = ChangeLog(**state.get(BUNDLE_DIFFS_OBJ))
+    data: list[dict[str, str]] = []
+    for item in change_log.items:
+        change_log_item = ChangeLogItem(**item)
+        commit = change_log_item.commit
+        covered_change_types_descriptions = [
+            ct.description
+            for ct in change_types
+            if ct.name in change_log_item.change_types
+        ]
+        item = {
+            "commit": f"[{commit[:7]}]({repo_url}/commit/{commit})",
+            "merged_at": change_log_item.merged_at,
+            "apps": ", ".join(change_log_item.apps),
+            "changes": ", ".join(covered_change_types_descriptions),
+        }
+        data.append(item)
+
+    # TODO(mafriedm): Fix this
+    ctx.obj["options"]["sort"] = False
+    columns = ["commit", "merged_at", "apps", "changes"]
+    print_output(ctx.obj["options"], data, columns)
+
+
+@root.group(name="set")
+@output
+@click.pass_context
+def set_command(ctx, output):
+    ctx.obj["output"] = output
+
+
+@set_command.command()
+@click.argument("workspace")
+@click.argument("usergroup")
+@click.argument("username")
+@click.pass_context
+def slack_usergroup(ctx, workspace, usergroup, username):
+    """Update users in a slack usergroup.
+    Use an org_username as the username.
+    To empty a slack usergroup, pass '' (empty string) as the username.
+    """
+    settings = queries.get_app_interface_settings()
+    slack = slackapi_from_queries("qontract-cli")
+    ugid = slack.get_usergroup_id(usergroup)
+    if username:
+        mail_address = settings["smtp"]["mailAddress"]
+        users = [slack.get_user_id_by_name(username, mail_address)]
+    else:
+        users = [slack.get_random_deleted_user()]
+    slack.update_usergroup_users(ugid, users)
+
+
+@set_command.command()
+@click.argument("org_name")
+@click.argument("cluster_name")
+@click.pass_context
+def cluster_admin(ctx, org_name, cluster_name):
+    settings = queries.get_app_interface_settings()
+    ocms = [
+        o for o in queries.get_openshift_cluster_managers() if o["name"] == org_name
+    ]
+    ocm_map = OCMMap(ocms=ocms, settings=settings)
+    ocm = ocm_map[org_name]
+    enabled = ocm.is_cluster_admin_enabled(cluster_name)
+    if not enabled:
+        ocm.enable_cluster_admin(cluster_name)
+
+
+@root.group()
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+@click.pass_context
+def state(ctx):
+    pass
+
+
+@state.command()
+@click.argument("integration", default="")
+@click.pass_context
+def ls(ctx, integration):
+    state = init_state(integration=integration)
+    keys = state.ls()
+    # if integration in not defined the 2th token will be the integration name
+    key_index = 1 if integration else 2
+    table_content = [
+        {
+            "integration": integration or k.split("/")[1],
+            "key": "/".join(k.split("/")[key_index:]),
         }
         for k in keys
     ]
@@ -2114,11 +3177,11 @@ def ls(ctx, integration):
     )
 
 
-@state.command()  # type: ignore
+@state.command(name="get")
 @click.argument("integration")
 @click.argument("key")
 @click.pass_context
-def get(ctx, integration, key):
+def state_get(ctx, integration, key):
     state = init_state(integration=integration)
     value = state.get(key)
     print(value)
@@ -2133,12 +3196,12 @@ def add(ctx, integration, key):
     state.add(key)
 
 
-@state.command()  # type: ignore
+@state.command(name="set")
 @click.argument("integration")
 @click.argument("key")
 @click.argument("value")
 @click.pass_context
-def set(ctx, integration, key, value):
+def state_set(ctx, integration, key, value):
     state = init_state(integration=integration)
     state.add(key, value=value, force=True)
 
@@ -2152,6 +3215,259 @@ def rm(ctx, integration, key):
     state.rm(key)
 
 
+@root.group()
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+@click.pass_context
+def early_exit_cache(ctx):
+    pass
+
+
+@early_exit_cache.command(name="head")
+@click.option(
+    "-i",
+    "--integration",
+    help="Integration name.",
+    required=True,
+)
+@click.option(
+    "-v",
+    "--integration-version",
+    help="Integration version.",
+    required=True,
+)
+@click.option(
+    "--dry-run/--no-dry-run",
+    help="",
+    default=False,
+)
+@click.option(
+    "-c",
+    "--cache-source",
+    help="Cache source. It should be a JSON string.",
+    required=True,
+)
+@click.option(
+    "-s",
+    "--shard",
+    help="Shard",
+    default="",
+)
+@click.pass_context
+def early_exit_cache_head(
+    ctx,
+    integration,
+    integration_version,
+    dry_run,
+    cache_source,
+    shard,
+):
+    with EarlyExitCache.build() as cache:
+        cache_key = CacheKey(
+            integration=integration,
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=json.loads(cache_source),
+            shard=shard,
+        )
+        print(f"cache_source_digest: {cache_key.cache_source_digest}")
+        result = cache.head(cache_key)
+        print(result)
+
+
+@early_exit_cache.command(name="get")
+@click.option(
+    "-i",
+    "--integration",
+    help="Integration name.",
+    required=True,
+)
+@click.option(
+    "-v",
+    "--integration-version",
+    help="Integration version.",
+    required=True,
+)
+@click.option(
+    "--dry-run/--no-dry-run",
+    help="",
+    default=False,
+)
+@click.option(
+    "-c",
+    "--cache-source",
+    help="Cache source. It should be a JSON string.",
+    required=True,
+)
+@click.option(
+    "-s",
+    "--shard",
+    help="Shard",
+    default="",
+)
+@click.pass_context
+def early_exit_cache_get(
+    ctx,
+    integration,
+    integration_version,
+    dry_run,
+    cache_source,
+    shard,
+):
+    with EarlyExitCache.build() as cache:
+        cache_key = CacheKey(
+            integration=integration,
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=json.loads(cache_source),
+            shard=shard,
+        )
+        value = cache.get(cache_key)
+        print(value)
+
+
+@early_exit_cache.command(name="set")
+@click.option(
+    "-i",
+    "--integration",
+    help="Integration name.",
+    required=True,
+)
+@click.option(
+    "-v",
+    "--integration-version",
+    help="Integration version.",
+    required=True,
+)
+@click.option(
+    "--dry-run/--no-dry-run",
+    help="",
+    default=False,
+)
+@click.option(
+    "-c",
+    "--cache-source",
+    help="Cache source. It should be a JSON string.",
+    required=True,
+)
+@click.option(
+    "-s",
+    "--shard",
+    help="Shard",
+    default="",
+)
+@click.option(
+    "-p",
+    "--payload",
+    help="Payload in Cache value. It should be a JSON string.",
+    required=True,
+)
+@click.option(
+    "-l",
+    "--log-output",
+    help="Log output.",
+    default="",
+)
+@click.option(
+    "-a",
+    "--applied-count",
+    help="Log output.",
+    default=0,
+    type=int,
+)
+@click.option(
+    "-t",
+    "--ttl",
+    help="TTL, in seconds.",
+    default=60,
+    type=int,
+)
+@click.option(
+    "-d",
+    "--latest-cache-source-digest",
+    help="Latest cache source digest.",
+    default="",
+)
+@click.pass_context
+def early_exit_cache_set(
+    ctx,
+    integration,
+    integration_version,
+    dry_run,
+    cache_source,
+    shard,
+    payload,
+    log_output,
+    applied_count,
+    ttl,
+    latest_cache_source_digest,
+):
+    with EarlyExitCache.build() as cache:
+        cache_key = CacheKey(
+            integration=integration,
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source=json.loads(cache_source),
+            shard=shard,
+        )
+        cache_value = CacheValue(
+            payload=json.loads(payload),
+            log_output=log_output,
+            applied_count=applied_count,
+        )
+        cache.set(cache_key, cache_value, ttl, latest_cache_source_digest)
+
+
+@early_exit_cache.command(name="delete")
+@click.option(
+    "-i",
+    "--integration",
+    help="Integration name.",
+    required=True,
+)
+@click.option(
+    "-v",
+    "--integration-version",
+    help="Integration version.",
+    required=True,
+)
+@click.option(
+    "--dry-run/--no-dry-run",
+    help="",
+    default=False,
+)
+@click.option(
+    "-d",
+    "--cache-source-digest",
+    help="Cache source digest.",
+    required=True,
+)
+@click.option(
+    "-s",
+    "--shard",
+    help="Shard",
+    default="",
+)
+@click.pass_context
+def early_exit_cache_delete(
+    ctx,
+    integration,
+    integration_version,
+    dry_run,
+    cache_source_digest,
+    shard,
+):
+    with EarlyExitCache.build() as cache:
+        cache_key_with_digest = CacheKeyWithDigest(
+            integration=integration,
+            integration_version=integration_version,
+            dry_run=dry_run,
+            cache_source_digest=cache_source_digest,
+            shard=shard,
+        )
+        cache.delete(cache_key_with_digest)
+        print("deleted")
+
+
 @root.command()
 @click.argument("cluster")
 @click.argument("namespace")
@@ -2205,6 +3521,13 @@ def template(ctx, cluster, namespace, kind, name, path, secret_reader):
 
 
 @root.command()
+@binary(["promtool"])
+@binary_version(
+    "promtool",
+    ["--version"],
+    promtool.PROMTOOL_VERSION_REGEX,
+    promtool.PROMTOOL_VERSION,
+)
 @click.argument("path")
 @click.argument("cluster")
 @click.option(
@@ -2231,7 +3554,7 @@ def run_prometheus_test(ctx, path, cluster, namespace, secret_reader):
 
     namespace_with_prom_rules, _ = orb.get_namespaces(
         ["prometheus-rule"],
-        cluster_name=cluster,
+        cluster_names=[cluster] if cluster else [],
         namespace_name=namespace,
     )
 
@@ -2260,113 +3583,10 @@ def run_prometheus_test(ctx, path, cluster, namespace, secret_reader):
 
 
 @root.command()
-@click.argument("path")
-@click.argument("cluster")
-@click.option(
-    "-n",
-    "--namespace",
-    default="openshift-customer-monitoring",
-    help="Cluster namespace where the rules are deployed. It defaults to "
-    "openshift-customer-monitoring.",
+@binary(["amtool"])
+@binary_version(
+    "amtool", ["--version"], amtool.AMTOOL_VERSION_REGEX, amtool.AMTOOL_VERSION
 )
-@click.option(
-    "-s",
-    "--secret-reader",
-    default="vault",
-    help="Location to read secrets.",
-    type=click.Choice(["config", "vault"]),
-)
-@click.pass_context
-def run_prometheus_test_old(ctx, path, cluster, namespace, secret_reader):
-    """Run prometheus tests in PATH loading associated rules from CLUSTER."""
-    gqlapi = gql.get_api()
-
-    if path.startswith("resources"):
-        path = path.replace("resources", "", 1)
-
-    try:
-        resource = gqlapi.get_resource(path)
-    except gql.GqlGetResourceError as e:
-        print(f"Error in provided PATH: {e}.")
-        sys.exit(1)
-
-    test = resource["content"]
-    data = get_data_from_jinja_test_template(test, ["rule_files", "target_clusters"])
-    target_clusters = data["target_clusters"]
-    if len(target_clusters) > 0 and cluster not in target_clusters:
-        print(
-            f"Skipping test: {path}, cluster {cluster} not in target_clusters {target_clusters}"
-        )
-    rule_files = data["rule_files"]
-    if not rule_files:
-        print(f"Cannot parse test in {path}.")
-        sys.exit(1)
-
-    if len(rule_files) > 1:
-        print("Only 1 rule file per test")
-        sys.exit(1)
-
-    rule_file_path = rule_files[0]
-
-    namespace_info = [
-        n
-        for n in gqlapi.query(orb.NAMESPACES_QUERY)["namespaces"]
-        if n["cluster"]["name"] == cluster and n["name"] == namespace
-    ]
-    if len(namespace_info) != 1:
-        print(f"{cluster}/{namespace} does not exist.")
-        sys.exit(1)
-
-    settings = queries.get_app_interface_settings()
-    settings["vault"] = secret_reader == "vault"
-
-    ni = namespace_info[0]
-    ob.aggregate_shared_resources(ni, "openshiftResources")
-    openshift_resources = ni.get("openshiftResources")
-    rule_spec = {}
-    for r in openshift_resources:
-        resource_path = r.get("resource", {}).get("path")
-        if resource_path != rule_file_path:
-            continue
-
-        if "add_path_to_prom_rules" not in r:
-            r["add_path_to_prom_rules"] = False
-
-        openshift_resource = orb.fetch_openshift_resource(r, ni, settings)
-        if openshift_resource.kind.lower() != "prometheusrule":
-            print(f"Object in {rule_file_path} is not a PrometheusRule.")
-            sys.exit(1)
-
-        rule_spec = openshift_resource.body["spec"]
-        variables = json.loads(r.get("variables") or "{}")
-        variables["resource"] = r
-        break
-
-    if not rule_spec:
-        print(
-            f"Rules file referenced in {path} does not exist in namespace "
-            f"{namespace} from cluster {cluster}."
-        )
-        sys.exit(1)
-
-    test_yaml_spec = yaml.safe_load(
-        orb.process_extracurlyjinja2_template(
-            body=test, vars=variables, settings=settings
-        )
-    )
-    test_yaml_spec.pop("$schema")
-
-    result = promtool.run_test(
-        test_yaml_spec=test_yaml_spec, rule_files={rule_file_path: rule_spec}
-    )
-
-    print(result)
-
-    if not result:
-        sys.exit(1)
-
-
-@root.command()
 @click.argument("cluster")
 @click.argument("namespace")
 @click.argument("rules_path")
@@ -2422,7 +3642,6 @@ def alert_to_receiver(
     secret_reader,
     additional_label,
 ):
-
     additional_labels = {}
     for al in additional_label:
         try:
@@ -2500,28 +3719,26 @@ def alert_to_receiver(
     for group in rule_spec["groups"]:
         for rule in group["rules"]:
             try:
+                # alertname label is added automatically by Prometheus.
                 alert_labels.append(
-                    {
-                        "name": rule["alert"],
-                        "labels": rule["labels"] | additional_labels,
-                    }
+                    {"alertname": rule["alert"]} | rule["labels"] | additional_labels
                 )
             except KeyError:
                 print("Skipping rule with no alert and/or labels", file=sys.stderr)
 
     if alert_name:
-        alert_labels = [al for al in alert_labels if al["name"] == alert_name]
+        alert_labels = [al for al in alert_labels if al["alertname"] == alert_name]
 
         if not alert_labels:
             print(f"Cannot find alert {alert_name} in rules {rules_path}")
             sys.exit(1)
 
     for al in alert_labels:
-        result = amtool.config_routes_test(am_config, al["labels"])
+        result = amtool.config_routes_test(am_config, al)
         if not result:
             print(f"Error running amtool: {result}")
             sys.exit(1)
-        print("|".join([al["name"], str(result)]))
+        print("|".join([al["alertname"], str(result)]))
 
 
 @root.command()
@@ -2530,7 +3747,7 @@ def alert_to_receiver(
 @click.option("--env-name", default=None, help="environment to use for parameters.")
 @click.pass_context
 def saas_dev(ctx, app_name=None, saas_file_name=None, env_name=None) -> None:
-    if env_name in [None, ""]:
+    if not env_name:
         print("env-name must be defined")
         return
     saas_files = get_saas_files(saas_file_name, env_name, app_name)
@@ -2544,21 +3761,17 @@ def saas_dev(ctx, app_name=None, saas_file_name=None, env_name=None) -> None:
                 if target.namespace.environment.name != env_name:
                     continue
 
-                parameters: dict[str, Any] = {}
-                parameters.update(target.namespace.environment.parameters or {})
-                parameters.update(saas_file.parameters or {})
-                parameters.update(rt.parameters or {})
-                parameters.update(target.parameters or {})
-
-                for replace_key, replace_value in parameters.items():
-                    if not isinstance(replace_value, str):
-                        continue
-                    replace_pattern = "${" + replace_key + "}"
-                    for k, v in parameters.items():
-                        if not isinstance(v, str):
-                            continue
-                        if replace_pattern in v:
-                            parameters[k] = v.replace(replace_pattern, replace_value)
+                parameters = TargetSpec(
+                    saas_file=saas_file,
+                    resource_template=rt,
+                    target=target,
+                    # process_template options
+                    image_auth=None,  # type: ignore[arg-type]
+                    hash_length=None,  # type: ignore[arg-type]
+                    github=None,  # type: ignore[arg-type]
+                    target_config_hash=None,  # type: ignore[arg-type]
+                    secret_reader=None,  # type: ignore[arg-type]
+                ).parameters()
 
                 parameters_cmd = ""
                 for k, v in parameters.items():
@@ -2581,7 +3794,7 @@ def saas_dev(ctx, app_name=None, saas_file_name=None, env_name=None) -> None:
 @click.option("--app-name", default=None, help="app to act on.")
 @click.pass_context
 def saas_targets(
-    ctx, saas_file_name: Optional[str] = None, app_name: Optional[str] = None
+    ctx, saas_file_name: str | None = None, app_name: str | None = None
 ) -> None:
     """Resolve namespaceSelectors and print all resulting targets of a saas file."""
     console = Console()
@@ -2613,7 +3826,7 @@ def saas_targets(
                 if target.parameters:
                     param_table = Table("Key", "Value", box=box.MINIMAL)
                     for k, v in target.parameters.items():
-                        param_table.add_row(k, v)
+                        param_table.add_row(k, str(v))
                     info.add_row("Parameters", param_table)
 
                 if target.secret_parameters:
@@ -2770,6 +3983,51 @@ def gpg_encrypt(
     ).execute()
 
 
+@root.command()
+@click.option("--channel", help="the channel that state is part of")
+@click.option("--sha", help="the commit sha we want state for")
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+def get_promotion_state(channel: str, sha: str):
+    from tools.saas_promotion_state.saas_promotion_state import (
+        SaasPromotionState,
+    )
+
+    bucket = os.environ.get("APP_INTERFACE_STATE_BUCKET")
+    region = os.environ.get("APP_INTERFACE_STATE_BUCKET_REGION", "us-east-1")
+    promotion_state = SaasPromotionState.create(promotion_state=None, saas_files=None)
+    for publisher_id, state in promotion_state.get(channel=channel, sha=sha).items():
+        print()
+        if not state:
+            print(f"No state found for {publisher_id=}")
+        else:
+            print(f"{publisher_id=}")
+            print(
+                f"State link: https://{region}.console.aws.amazon.com/s3/object/{bucket}?region={region}&bucketType=general&prefix=state/openshift-saas-deploy/promotions_v2/{channel}/{publisher_id}/{sha}"
+            )
+            print(f"Content: {state}")
+
+
+@root.command()
+@click.option("--channel", help="the channel that state is part of")
+@click.option("--sha", help="the commit sha we want state for")
+@click.option("--publisher-id", help="the publisher id we want state for")
+@environ(["APP_INTERFACE_STATE_BUCKET"])
+def mark_promotion_state_successful(channel: str, sha: str, publisher_id: str):
+    from tools.saas_promotion_state.saas_promotion_state import (
+        SaasPromotionState,
+    )
+
+    promotion_state = SaasPromotionState.create(promotion_state=None, saas_files=None)
+    print(f"Current states for {publisher_id=}")
+    print(promotion_state.get(channel=channel, sha=sha).get(publisher_id, None))
+    print()
+    print("Pushing new state ...")
+    promotion_state.set_successful(channel=channel, sha=sha, publisher_uid=publisher_id)
+    print()
+    print(f"New state for {publisher_id=}")
+    print(promotion_state.get(channel=channel, sha=sha).get(publisher_id, None))
+
+
 @root.command()
 @click.option("--change-type-name")
 @click.option("--role-name")
@@ -2785,5 +4043,375 @@ def test_change_type(change_type_name: str, role_name: str, app_interface_path:
     tester.test_change_type_in_context(change_type_name, role_name, app_interface_path)
 
 
+@root.group()
+@click.pass_context
+def sso_client(ctx):
+    """SSO client commands"""
+
+
+@sso_client.command()
+@click.argument("client-name", required=True)
+@click.option(
+    "--contact-email",
+    default="sd-app-sre+auth@redhat.com",
+    help="Specify the contact email address",
+    required=True,
+    show_default=True,
+)
+@click.option(
+    "--keycloak-instance-vault-path",
+    help="Path to the keycloak secret in vault",
+    default="app-sre/creds/rhidp/auth.redhat.com",
+    required=True,
+    show_default=True,
+)
+@click.option(
+    "--request-uri",
+    help="Specify an allowed request URL; first one will be used as the initial one URL. Can be specified multiple times",
+    multiple=True,
+    required=True,
+    prompt=True,
+)
+@click.option(
+    "--redirect-uri",
+    help="Specify an allowed redirect URL. Can be specified multiple times",
+    multiple=True,
+    required=True,
+    prompt=True,
+)
+@click.pass_context
+def create(
+    ctx,
+    client_name: str,
+    contact_email: str,
+    keycloak_instance_vault_path: str,
+    request_uri: tuple[str],
+    redirect_uri: tuple[str],
+) -> None:
+    """Create a new SSO client"""
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
+    keycloak_secret = secret_reader.read_all({"path": keycloak_instance_vault_path})
+    keycloak_api = KeycloakAPI(
+        url=keycloak_secret["url"],
+        initial_access_token=keycloak_secret["initial-access-token"],
+    )
+    sso_client = keycloak_api.register_client(
+        client_name=client_name,
+        redirect_uris=redirect_uri,
+        initiate_login_uri=request_uri[0],
+        request_uris=request_uri,
+        contacts=[contact_email],
+    )
+    click.secho(
+        "SSO client created successfully. Please save the following JSON in Vault!",
+        bg="red",
+        fg="white",
+    )
+    print(sso_client.json(by_alias=True, indent=2))
+
+
+@sso_client.command()
+@click.argument("sso-client-vault-secret-path", required=True)
+@click.pass_context
+def remove(ctx, sso_client_vault_secret_path: str):
+    """Remove an existing SSO client"""
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
+    sso_client = SSOClient(
+        **secret_reader.read_all({"path": sso_client_vault_secret_path})
+    )
+    keycloak_api = KeycloakAPI()
+    keycloak_api.delete_client(
+        registration_client_uri=sso_client.registration_client_uri,
+        registration_access_token=sso_client.registration_access_token,
+    )
+    click.secho(
+        "SSO client removed successfully. Please remove the secret from Vault!",
+        bg="red",
+        fg="white",
+    )
+
+
+@root.group()
+@click.option(
+    "--provision-provider",
+    required=True,
+    help="externalResources.provider",
+    default="aws",
+)
+@click.option(
+    "--provisioner",
+    required=True,
+    help="externalResources.provisioner.name. E.g. app-sre-stage",
+    prompt=True,
+)
+@click.option(
+    "--provider",
+    required=True,
+    help="externalResources.resources.provider. E.g. rds, msk, ...",
+    prompt=True,
+)
+@click.option(
+    "--identifier",
+    required=True,
+    help="externalResources.resources.identifier. E.g. erv2-example",
+    prompt=True,
+)
+@click.pass_context
+def external_resources(
+    ctx, provision_provider: str, provisioner: str, provider: str, identifier: str
+):
+    """External resources commands"""
+    ctx.obj["provision_provider"] = provision_provider
+    ctx.obj["provisioner"] = provisioner
+    ctx.obj["provider"] = provider
+    ctx.obj["identifier"] = identifier
+    vault_settings = get_app_interface_vault_settings()
+    ctx.obj["secret_reader"] = create_secret_reader(use_vault=vault_settings.vault)
+
+
+@external_resources.command()
+@click.pass_context
+def get_input(ctx):
+    """Gets the input data for an external resource asset. Input data is what is used
+    in the Reconciliation Job to manage the resource."""
+    erv2cli = Erv2Cli(
+        provision_provider=ctx.obj["provision_provider"],
+        provisioner=ctx.obj["provisioner"],
+        provider=ctx.obj["provider"],
+        identifier=ctx.obj["identifier"],
+        secret_reader=ctx.obj["secret_reader"],
+    )
+    print(erv2cli.input_data)
+
+
+@external_resources.command()
+@click.pass_context
+def request_reconciliation(ctx):
+    """Marks a resource as it needs to get reconciled. The itegration will reconcile the resource at
+    its next iteration."""
+    erv2cli = Erv2Cli(
+        provision_provider=ctx.obj["provision_provider"],
+        provisioner=ctx.obj["provisioner"],
+        provider=ctx.obj["provider"],
+        identifier=ctx.obj["identifier"],
+        secret_reader=ctx.obj["secret_reader"],
+    )
+    erv2cli.reconcile()
+
+
+@external_resources.command()
+@binary(["terraform", "docker"])
+@binary_version("terraform", ["version"], TERRAFORM_VERSION_REGEX, TERRAFORM_VERSION)
+@click.option(
+    "--dry-run/--no-dry-run",
+    help="Enable/Disable dry-run. Default: dry-run enabled!",
+    default=True,
+)
+@click.option(
+    "--skip-build/--no-skip-build",
+    help="Skip/Do not skip the terraform and CDKTF builds. Default: build everything!",
+    default=False,
+)
+@click.pass_context
+def migrate(ctx, dry_run: bool, skip_build: bool) -> None:
+    """Migrate an existing external resource managed by terraform-resources to ERv2.
+
+
+    E.g: qontract-reconcile --config=<config> external-resources migrate aws app-sre-stage rds dashdotdb-stage
+    """
+    if ctx.obj["provider"] == "rds":
+        # The "random_password" is not an AWS resource. It's just in the outputs and can't be migrated :(
+        raise NotImplementedError("RDS migration is not supported yet!")
+
+    if not Confirm.ask(
+        dedent("""
+            Please ensure [red]terraform-resources[/] is disabled before proceeding!
+
+            Do you want to proceed?"""),
+        default=True,
+    ):
+        sys.exit(0)
+
+    # use a temporary directory in $HOME. The MacOS colima default configuration allows docker mounts from $HOME.
+    tempdir = Path.home() / ".erv2-migration"
+    rich_print(f"Using temporary directory: [b]{tempdir}[/]")
+    tempdir.mkdir(exist_ok=True)
+    temp_erv2 = Path(tempdir) / "erv2"
+    temp_erv2.mkdir(exist_ok=True)
+    temp_tfr = tempdir / "terraform-resources"
+    temp_tfr.mkdir(exist_ok=True)
+
+    with progress_spinner() as progress:
+        with task(progress, "Preparing AWS credentials for CDKTF and local terraform"):
+            # prepare AWS credentials for CDKTF and local terraform
+            credentials_file = tempdir / "credentials"
+            credentials_file.write_text(
+                ctx.obj["secret_reader"].read_with_parameters(
+                    path=f"app-sre/external-resources/{ctx.obj['provisioner']}",
+                    field="credentials",
+                    format=None,
+                    version=None,
+                )
+            )
+        os.environ["AWS_SHARED_CREDENTIALS_FILE"] = str(credentials_file)
+
+        erv2cli = Erv2Cli(
+            provision_provider=ctx.obj["provision_provider"],
+            provisioner=ctx.obj["provisioner"],
+            provider=ctx.obj["provider"],
+            identifier=ctx.obj["identifier"],
+            secret_reader=ctx.obj["secret_reader"],
+            temp_dir=temp_erv2,
+            progress_spinner=progress,
+        )
+
+        with task(progress, "(erv2) Building the terraform configuration"):
+            if not skip_build:
+                # build the CDKTF output
+                erv2cli.build_cdktf(credentials_file)
+            erv2_tf_cli = TerraformCli(
+                temp_erv2, dry_run=dry_run, progress_spinner=progress
+            )
+            if not skip_build:
+                erv2_tf_cli.init()
+
+        with task(
+            progress, "(terraform-resources) Building the terraform configuration"
+        ):
+            # build the terraform-resources output
+            conf_tf = temp_tfr / "conf.tf.json"
+            if not skip_build:
+                tfr.run(
+                    dry_run=True,
+                    print_to_file=str(conf_tf),
+                    account_name=[ctx.obj["provisioner"]],
+                )
+            # remove comments
+            conf_tf.write_text(
+                "\n".join(
+                    line
+                    for line in conf_tf.read_text().splitlines()
+                    if not line.startswith("#")
+                )
+            )
+            tfr_tf_cli = TerraformCli(
+                temp_tfr, dry_run=dry_run, progress_spinner=progress
+            )
+            if not skip_build:
+                tfr_tf_cli.init()
+
+    with progress_spinner() as progress:
+        # start a new spinner instance for clean output
+        erv2_tf_cli.progress_spinner = progress
+        with task(
+            progress,
+            "Migrating the resources from terraform-resources to ERv2",
+        ):
+            if ctx.obj["provider"] == "elasticache":
+                # Elasticache migration is a bit different
+                erv2_tf_cli.migrate_elasticache_resources(source=tfr_tf_cli)
+            else:
+                erv2_tf_cli.migrate_resources(source=tfr_tf_cli)
+
+    rich_print(f"[b red]Please remove the temporary directory ({tempdir}) manually!")
+
+
+@external_resources.command()
+@binary(["docker"])
+@click.pass_context
+def debug_shell(ctx) -> None:
+    """Enter an ERv2 debug shell to manually migrate resources."""
+    # use a temporary directory in $HOME. The MacOS colima default configuration allows docker mounts from $HOME.
+    with tempfile.TemporaryDirectory(dir=Path.home(), prefix="erv2-debug.") as _tempdir:
+        tempdir = Path(_tempdir)
+        with progress_spinner() as progress:
+            with task(progress, "Preparing environment ..."):
+                credentials_file = tempdir / "credentials"
+                credentials_file.write_text(
+                    ctx.obj["secret_reader"].read_with_parameters(
+                        path=f"app-sre/external-resources/{ctx.obj['provisioner']}",
+                        field="credentials",
+                        format=None,
+                        version=None,
+                    )
+                )
+            os.environ["AWS_SHARED_CREDENTIALS_FILE"] = str(credentials_file)
+
+        erv2cli = Erv2Cli(
+            provision_provider=ctx.obj["provision_provider"],
+            provisioner=ctx.obj["provisioner"],
+            provider=ctx.obj["provider"],
+            identifier=ctx.obj["identifier"],
+            secret_reader=ctx.obj["secret_reader"],
+            temp_dir=tempdir,
+            progress_spinner=progress,
+        )
+        erv2cli.enter_shell(credentials_file)
+
+
+@get.command(help="Get all container images in app-interface defined namespaces")
+@cluster_name
+@namespace_name
+@thread_pool_size()
+@use_jump_host()
+@click.option("--exclude-pattern", help="Exclude images that match this pattern")
+@click.option("--include-pattern", help="Only include images that match this pattern")
+@click.pass_context
+def container_images(
+    ctx,
+    cluster_name,
+    namespace_name,
+    thread_pool_size,
+    use_jump_host,
+    exclude_pattern,
+    include_pattern,
+):
+    from tools.cli_commands.container_images_report import get_all_pods_images
+
+    results = get_all_pods_images(
+        cluster_name=cluster_name,
+        namespace_name=namespace_name,
+        thread_pool_size=thread_pool_size,
+        use_jump_host=use_jump_host,
+        exclude_pattern=exclude_pattern,
+        include_pattern=include_pattern,
+    )
+
+    if ctx.obj["options"]["output"] == "md":
+        json_table = {
+            "filter": True,
+            "fields": [
+                {"key": "name", "sortable": True},
+                {"key": "namespaces", "sortable": True},
+                {"key": "count", "sortable": True},
+            ],
+            "items": results,
+        }
+
+        print(
+            f"""
+You can view the source of this Markdown to extract the JSON data.
+
+{len(results)} container images found.
+
+```json:table
+{json.dumps(json_table)}
+```
+            """
+        )
+    else:
+        columns = [
+            "name",
+            "namespaces",
+            "count",
+        ]
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], results, columns)
+
+
 if __name__ == "__main__":
     root()  # pylint: disable=no-value-for-parameter