qontract-reconcile 0.10.1rc58__py3-none-any.whl → 0.10.1rc60__py3-none-any.whl

{qontract_reconcile-0.10.1rc58.dist-info → qontract_reconcile-0.10.1rc60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc58
+Version: 0.10.1rc60
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc58.dist-info → qontract_reconcile-0.10.1rc60.dist-info}/RECORD

@@ -7,7 +7,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=i6bSWnlH9fh14P14PjVhFLwNl-q3fD733_rXKM_O51c,2992
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=figtZRuWUvdpdSnkhAqeGvO5dI02TT6J3heyeFhlwqM,5016
-reconcile/cli.py,sha256=KVARhSmgO5IvCmlZwD4q6ZziFMtEPbXkc7O_qGa-hco,71195
+reconcile/cli.py,sha256=c3tSmmo4ZwH23ve5iHDRA7zFLJvyG4DPw9G-LdWJkN0,71473
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=0xg_d8dwd36Y8GY1mE-LLO1LQpPEMM77bzAfc_KdgzU,4870
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=Ca75-OQiu5HeA8Q6zQpEYuhyCSjeuWe99K4y9ipTORM,4032
@@ -94,7 +94,7 @@ reconcile/quay_mirror.py,sha256=9pwl1gLzRpsVXF5yPULM4ET_C5F8_xPmH8Mv8AS2AfI,1340
 reconcile/quay_mirror_org.py,sha256=E1OdRe-ppxTkNCwu20iVRhEdG1fPDBroLY02NgiMN7c,10381
 reconcile/quay_permissions.py,sha256=_3PCWjNWoU7VHlYgHzUevvL_jJmEMsWfXV_nzjeiyhU,4099
 reconcile/quay_repos.py,sha256=7609RBVQihis96FNOOe-i9tCTYwcTVy4WpKAL6HpnkU,7031
-reconcile/queries.py,sha256=rdmH3Pyh2KjX3ZMcYRMMSODO0AntdY7Iju0zdxd2QkI,49222
+reconcile/queries.py,sha256=UMUfwqFmujdUJ9z1X_RN-Mktjag144An6jRgoZza57o,49432
 reconcile/query_validator.py,sha256=oLEZIAsQCzxmmZ7b9dSw-OKuEjpI1dbVu4XfCfjpmi8,1503
 reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
 reconcile/resource_scraper.py,sha256=vo1N9vLJCYWvXlTwFRIpEuWjx_39ZV9zxJlpoPq4g3U,2330
@@ -127,16 +127,18 @@ reconcile/aus/models.py,sha256=kRrs276iJClnNnEeWnJma2Gvx8E9kes77I_XbgKoVzk,4722
 reconcile/aus/ocm_addons_upgrade_scheduler_org.py,sha256=UjKFUeomkVpwVVkejPfNBMN7CQywIs5uOipUckfI8Y0,7659
 reconcile/aus/ocm_upgrade_scheduler.py,sha256=ndruHjyheMzjZgcIQQEG7qjLnhKL4pF_kIExxBrYZZM,5153
 reconcile/aus/ocm_upgrade_scheduler_org.py,sha256=tH0oQFJsO4UrEYspqiq-ne6-ULXA4NLY3g3hxaJKGEA,2867
+reconcile/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/aws_ami_cleanup/integration.py,sha256=5-znzuQAGz9hcepBrVowzylqUQuAmVpAPRZKBl5QFZg,8752
 reconcile/change_owners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/change_owners/approver.py,sha256=L7XJWJ-rgn8BOmeMb6lBDV8lHFCUaNoHGDSD7OH03vA,2244
 reconcile/change_owners/bundle.py,sha256=VTRXDMlcNYykqCPRIcCDTCA188Ouxglt7Sxeu1qep4E,5102
-reconcile/change_owners/change_owners.py,sha256=0JRRPaaCCfgNpxwVfWDYCCYoa0II6OSZuiO6jpVwsLM,14392
+reconcile/change_owners/change_owners.py,sha256=WX-ekjI3IhdrJtvwieyeE32ho6cOsIOzFY3uzS25X5E,13268
 reconcile/change_owners/change_types.py,sha256=rWYmrw3rcE-lEZ-SQb3D38an5GhasaydpAtYpg5nPBU,28405
 reconcile/change_owners/changes.py,sha256=uGFukXgMkcO9xAXzF_LL0HLdNcQtSq0IyojsI1YIJTs,15889
 reconcile/change_owners/decision.py,sha256=uFuk06kzgifCg4ExaMRHEX6LIKnwaMkSQG8AeNG2uJo,4811
 reconcile/change_owners/diff.py,sha256=84pbx19C-QL1SgK16MxPqv3Ssiy1g8j0i978iSDb2kM,8840
 reconcile/change_owners/implicit_ownership.py,sha256=xga4X0OiZityQf5YRGLUSaIdr2Ij74H879-sjWzgTDk,4161
-reconcile/change_owners/self_service_roles.py,sha256=6xHBCaOeFjuFn5A3cBJZ77PdfkvOmYyHujuyohoEHb0,5173
+reconcile/change_owners/self_service_roles.py,sha256=Ya5NPHCFpzMCDmxHrrjueXCYAap7jXzm0woudUEH6lc,6754
 reconcile/change_owners/tester.py,sha256=BfJM8dsurubnyDF9UR3oV2N3ClyjLsqDNq0Hy_UdjOk,8949
 reconcile/cna/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/cna/client.py,sha256=t9gJDrKf4ApBlgu8c4QUbmzrYoSo1QPsnAGfucva2_U,1562
@@ -155,6 +157,8 @@ reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 reconcile/gql_definitions/advanced_upgrade_service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/advanced_upgrade_service/aus_clusters.py,sha256=eFo-836XT566MwEzlbb4W7eNdPmh_gUzDUUFwCzX7Kk,3731
 reconcile/gql_definitions/advanced_upgrade_service/aus_organization.py,sha256=rzxC3w1Bff3VSpI4zGN1wR5cSOmPxLYKBuwFGGwraPU,2872
+reconcile/gql_definitions/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/aws_ami_cleanup/asg_namespaces.py,sha256=hu0f4DQ5a6mP4L27DR9vbh01tuKYFS-vBE-PttOnFHE,3746
 reconcile/gql_definitions/change_owners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/change_owners/queries/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/change_owners/queries/change_types.py,sha256=LViPCCECF0BQLk9RoB_4uUrkqQ3dRo73bVtxElqYLxQ,4993
@@ -297,6 +301,7 @@ reconcile/test/fixtures.py,sha256=VhvLXH0AWXEyu3FgPp7bcSTPmDPfMEa2v-_9cd8dCmw,57
 reconcile/test/test_aggregated_list.py,sha256=iiWitQuNYC58aimWaiBoE4NROHjr1NCgQ91MnHEG_Ro,6412
 reconcile/test/test_amtool.py,sha256=vxRhGieeydMBOb9UI2ziMHjJa8puMeGNsUhGhy-yMnk,1032
 reconcile/test/test_auto_promoter.py,sha256=4EtLLN0FAJGJnFoSCRsB5hHyY2-H3GMfnn5-p7kNCTg,10340
+reconcile/test/test_aws_ami_cleanup.py,sha256=vHSldoVnZtfc1s_7KDh7GkzPTr7hF84ABj_vMNBR06s,8633
 reconcile/test/test_aws_ami_share.py,sha256=eSITdDoXs8mMY7P2lFxAX2DA0sJ9RW6D1tG8Rek0gLE,1981
 reconcile/test/test_aws_iam_keys.py,sha256=MfE9EvItyPNPAl5QaLlJFUvvrZFiar518TM2wWNjJn4,1829
 reconcile/test/test_aws_iam_password_reset.py,sha256=fnkqB90adR7W4L4saNdrtIiwnQB9bXgqJ9R1CKxjSnk,860
@@ -584,8 +589,8 @@ tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y
 tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_qontract_cli.py,sha256=awwTHEc2DWlykuqGIYM0WOBoSL0KRnOraCLk3C7izis,1401
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc58.dist-info/METADATA,sha256=oWz9xx5iFJYxcx-A0cQXP8M_NngpvZJY5z4m3EuADuA,2290
-qontract_reconcile-0.10.1rc58.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-qontract_reconcile-0.10.1rc58.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
-qontract_reconcile-0.10.1rc58.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc58.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc60.dist-info/METADATA,sha256=JAgKlKekEmzyQTqdv9oMiVAsoeAyIaDysoW4U1WavCw,2290
+qontract_reconcile-0.10.1rc60.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+qontract_reconcile-0.10.1rc60.dist-info/entry_points.txt,sha256=Af70EWPJxsTiCNF6gA-pWdw1A0Heqn-PZF-oBc5NmiU,302
+qontract_reconcile-0.10.1rc60.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc60.dist-info/RECORD,,

reconcile/aws_ami_cleanup/integration.py

@@ -0,0 +1,283 @@
+import logging
+import re
+import sys
+from collections.abc import (
+    Callable,
+    Mapping,
+)
+from datetime import (
+    datetime,
+    timedelta,
+)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Optional,
+)
+
+from botocore.exceptions import ClientError
+from pydantic import (
+    BaseModel,
+    Field,
+)
+
+from reconcile import queries
+from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
+    ASGImageGitV1,
+    ASGImageStaticV1,
+    NamespaceTerraformProviderResourceAWSV1,
+    NamespaceTerraformResourceASGV1,
+    NamespaceV1,
+)
+from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
+    query as query_asg_namespaces,
+)
+from reconcile.status import ExitCodes
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.utils import gql
+from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.defer import defer
+from reconcile.utils.parse_dhms_duration import dhms_to_seconds
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
+
+if TYPE_CHECKING:
+    from mypy_boto3_ec2 import EC2Client
+else:
+    EC2Client = object
+
+QONTRACT_INTEGRATION = "aws_ami_cleanup"
+MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
+
+
+class CannotCompareTagsError(Exception):
+    pass
+
+
+class AmiTag(BaseModel):
+    key: str = Field(alias="Key")
+    value: str = Field(alias="Value")
+
+    class Config:
+        allow_population_by_field_name = True
+        frozen = True
+
+
+class AWSAmi(BaseModel):
+    name: str
+    image_id: str
+    tags: set[AmiTag]
+    creation_date: datetime
+
+    class Config:
+        frozen = True
+
+
+class AIAmi(BaseModel):
+    identifier: str
+    tags: set[AmiTag]
+
+    class Config:
+        frozen = True
+
+
+def get_aws_amis(
+    aws_api: AWSApi,
+    ec2_client: EC2Client,
+    owner: str,
+    regex: str,
+    age_in_seconds: int,
+    utc_now: datetime,
+    region: str,
+) -> list[AWSAmi]:
+    """Get amis that match regex older than given age"""
+
+    images = aws_api.paginate(
+        ec2_client, "describe_images", "Images", {"Owners": [owner]}
+    )
+
+    pattern = re.compile(regex)
+    results = []
+    for i in images:
+        if not re.search(pattern, i["Name"]):
+            continue
+
+        creation_date = datetime.strptime(i["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
+        current_delta = utc_now - creation_date
+        delete_delta = timedelta(seconds=age_in_seconds)
+
+        if current_delta < delete_delta:
+            continue
+
+        # We have nothing to do with untagged AMIs since we will need tags to verify if AMI are
+        # in use or not.
+        if not i.get("Tags"):
+            continue
+
+        tags = {AmiTag(**tag) for tag in i.get("Tags")}
+        results.append(
+            AWSAmi(
+                name=i["Name"],
+                image_id=i["ImageId"],
+                tags=tags,
+                creation_date=creation_date,
+            )
+        )
+
+    return results
+
+
+def get_region(
+    cleanup: Mapping[str, Any],
+    account: Mapping[str, Any],
+) -> str:
+    """Defines the region to search for AMIs."""
+    region = cleanup.get("region") or account["resourcesDefaultRegion"]
+    if region not in account["supportedDeploymentRegions"]:
+        raise ValueError(f"region {region} is not supported in {account['name']}")
+
+    return region
+
+
+def get_app_interface_amis(
+    namespaces: Optional[list[NamespaceV1]], ts: Terrascript
+) -> list[AIAmi]:
+    """Returns all the ami referenced in ASGs in app-interface."""
+    app_interface_amis = []
+    for n in namespaces or []:
+        for er in n.external_resources or []:
+            if not isinstance(er, NamespaceTerraformProviderResourceAWSV1):
+                continue
+
+            for r in er.resources:
+                if not isinstance(r, NamespaceTerraformResourceASGV1):
+                    continue
+
+                tags = set()
+                for i in r.image:
+                    if isinstance(i, ASGImageGitV1):
+                        tags.add(
+                            AmiTag(
+                                key=i.tag_name,
+                                value=ts.get_commit_sha(i.dict(by_alias=True)),
+                            )
+                        )
+                    elif isinstance(i, ASGImageStaticV1):
+                        tags.add(AmiTag(key=i.tag_name, value=i.value))
+
+                app_interface_amis.append(AIAmi(identifier=r.identifier, tags=tags))
+
+    return app_interface_amis
+
+
+def check_aws_ami_in_use(
+    aws_ami: AWSAmi, app_interface_amis: list[AIAmi]
+) -> Optional[str]:
+    """Verifies if the given AWS ami is in use in a defined app-interface ASG."""
+    for ai_ami in app_interface_amis:
+        # This can happen if the ASG init template has changed over the time. We don't have a way
+        # to properly delete these automatically since we cannot assure they are not in use.
+        # The integration will fail in this case and these amis will need to be handled manually.
+        if len(ai_ami.tags) > len(aws_ami.tags):
+            raise CannotCompareTagsError(
+                f"{ai_ami.identifier} AI AMI has more tags than {aws_ami.image_id} AWS AMI"
+            )
+
+        if ai_ami.tags.issubset(aws_ami.tags):
+            return ai_ami.identifier
+
+    return None
+
+
+@defer
+def run(dry_run: bool, thread_pool_size: int, defer: Optional[Callable] = None) -> None:
+    exit_code = ExitCodes.SUCCESS
+
+    # We still use here a non-typed query; accounts is passed to AWSApi and Terrascript classes
+    # which contain a vast amount of magic based on keys from that dict. Since this integration
+    # cannot still be properly monitored (see https://issues.redhat.com/browse/APPSRE-7674),
+    # it's easy that it breaks without being noticed. Once it is properly monitored, this should
+    # be moved to a typed query.
+    cleanup_accounts = [
+        a
+        for a in queries.get_aws_accounts(terraform_state=True, cleanup=True)
+        if a.get("cleanup")
+    ]
+
+    vault_settings = get_app_interface_vault_settings()
+
+    ts = Terrascript(
+        QONTRACT_INTEGRATION,
+        "",
+        thread_pool_size,
+        cleanup_accounts,
+        settings=vault_settings.dict(by_alias=True),
+    )
+
+    gqlapi = gql.get_api()
+    namespaces = query_asg_namespaces(query_func=gqlapi.query).namespaces or []
+    app_interface_amis = get_app_interface_amis(namespaces, ts)
+
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    aws_api = AWSApi(1, cleanup_accounts, secret_reader=secret_reader, init_users=False)
+    if defer:  # defer is provided by the method decorator; this makes just mypy happy.
+        defer(aws_api.cleanup)
+
+    utc_now = datetime.utcnow()
+    for account in cleanup_accounts:
+        for cleanup in account["cleanup"]:
+            if cleanup["provider"] != "ami":
+                continue
+
+            region = get_region(cleanup, account)
+            regex = cleanup["regex"]
+            age_in_seconds = dhms_to_seconds(cleanup["age"])
+
+            session = aws_api.get_session(account["name"])
+            ec2_client = aws_api.get_session_client(session, "ec2", region)
+
+            amis = get_aws_amis(
+                aws_api=aws_api,
+                ec2_client=ec2_client,
+                owner=account["uid"],
+                regex=regex,
+                age_in_seconds=age_in_seconds,
+                utc_now=utc_now,
+                region=region,
+            )
+
+            for aws_ami in amis:
+                try:
+                    if identifier := check_aws_ami_in_use(aws_ami, app_interface_amis):
+                        logging.info(
+                            "Discarding ami %s with id %s as it is used in app-interface in %s",
+                            aws_ami.name,
+                            aws_ami.image_id,
+                            identifier,
+                        )
+                        continue
+                except CannotCompareTagsError as e:
+                    exit_code = ExitCodes.ERROR
+                    logging.error(e)
+                    continue
+
+                logging.info(
+                    "Deregistering image %s with id %s created in %s",
+                    aws_ami.name,
+                    aws_ami.image_id,
+                    aws_ami.creation_date,
+                )
+
+                try:
+                    ec2_client.deregister_image(
+                        ImageId=aws_ami.image_id, DryRun=dry_run
+                    )
+                except ClientError as e:
+                    if "DryRunOperation" in str(e):
+                        logging.info(e)
+                        continue
+                    raise
+
+    sys.exit(exit_code)

reconcile/change_owners/change_owners.py

@@ -29,12 +29,9 @@ from reconcile.change_owners.implicit_ownership import (
 )
 from reconcile.change_owners.self_service_roles import (
     cover_changes_with_self_service_roles,
+    fetch_self_service_roles,
 )
-from reconcile.gql_definitions.change_owners.queries import (
-    change_types,
-    self_service_roles,
-)
-from reconcile.gql_definitions.change_owners.queries.self_service_roles import RoleV1
+from reconcile.gql_definitions.change_owners.queries import change_types
 from reconcile.utils import gql
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.labels import (
@@ -78,33 +75,6 @@ def cover_changes(
     )
 
 
-def validate_self_service_role(role: RoleV1) -> None:
-    for ssc in role.self_service or []:
-        if ssc.change_type.context_schema:
-            # check that all referenced datafiles have a schema that
-            # is compatible with the change-type
-            incompatible_datafiles = [
-                df.path
-                for df in ssc.datafiles or []
-                if df.datafile_schema != ssc.change_type.context_schema
-            ]
-            if incompatible_datafiles:
-                raise ValueError(
-                    f"The datafiles {incompatible_datafiles} are not compatible with the "
-                    f"{ssc.change_type.name} change-types contextSchema {ssc.change_type.context_schema}"
-                )
-
-
-def fetch_self_service_roles(gql_api: gql.GqlApi) -> list[RoleV1]:
-    roles: list[RoleV1] = []
-    for r in self_service_roles.query(gql_api.query).roles or []:
-        if not r.self_service:
-            continue
-        validate_self_service_role(r)
-        roles.append(r)
-    return roles
-
-
 def fetch_change_type_processors(
     gql_api: gql.GqlApi, file_diff_resolver: FileDiffResolver
 ) -> list[ChangeTypeProcessor]:

reconcile/change_owners/self_service_roles.py

@@ -12,15 +12,62 @@ from reconcile.change_owners.change_types import (
     ChangeTypeProcessor,
 )
 from reconcile.change_owners.changes import BundleFileChange
+from reconcile.gql_definitions.change_owners.queries import self_service_roles
 from reconcile.gql_definitions.change_owners.queries.self_service_roles import (
     PermissionGitlabGroupMembershipV1,
     PermissionSlackUsergroupV1,
     RoleV1,
 )
+from reconcile.utils import gql
 
 
-class EmptySelfServiceRoleError(Exception):
-    pass
+class NoApproversInSelfServiceRoleError(Exception):
+    """
+    Thrown when a self-service role has no approvers
+    """
+
+
+class DatafileIncompatibleWithChangeTypeError(Exception):
+    """
+    Thrown when a datafile and a change type are hooked up
+    in a self-service role, but are not compatible schema wise.
+    """
+
+
+def fetch_self_service_roles(gql_api: gql.GqlApi) -> list[RoleV1]:
+    roles: list[RoleV1] = []
+    for r in self_service_roles.query(gql_api.query).roles or []:
+        if not r.self_service:
+            continue
+        validate_self_service_role(r)
+        roles.append(r)
+    return roles
+
+
+def validate_self_service_role(role: RoleV1) -> None:
+    """
+    Validate that a self-service role has approvers and that the referenced
+    change-types and datafiles/resources are compatible.
+    """
+    if not role.users and not role.bots:
+        raise NoApproversInSelfServiceRoleError(
+            f"The role {role.name} has no users or bots "
+            "to drive the self-service process. Add approvers to the roles."
+        )
+    for ssc in role.self_service or []:
+        if ssc.change_type.context_schema:
+            # check that all referenced datafiles have a schema that
+            # is compatible with the change-type
+            incompatible_datafiles = [
+                df.path
+                for df in ssc.datafiles or []
+                if df.datafile_schema != ssc.change_type.context_schema
+            ]
+            if incompatible_datafiles:
+                raise DatafileIncompatibleWithChangeTypeError(
+                    f"The datafiles {incompatible_datafiles} are not compatible with the "
+                    f"{ssc.change_type.name} change-types contextSchema {ssc.change_type.context_schema}"
+                )
 
 
 def cover_changes_with_self_service_roles(
@@ -66,11 +113,6 @@ def change_type_contexts_for_self_service_roles(
                         role_lookup[
                             (BundleFileType.RESOURCEFILE, res, ss.change_type.name)
                         ].append(r)
-    if orphaned_roles:
-        raise EmptySelfServiceRoleError(
-            f"The roles {', '.join([r.name for r in orphaned_roles])} have no users or bots "
-            "to drive the self-service process. Add approvers to the roles."
-        )
 
     # match every BundleChange with every relevant ChangeTypeV1
     change_type_contexts = []

reconcile/cli.py

@@ -966,6 +966,15 @@ def aws_ami_share(ctx):
     run_integration(reconcile.aws_ami_share, ctx.obj)
 
 
+@integration.command(short_help="Cleanup old and unused AMIs.")
+@threaded()
+@click.pass_context
+def aws_ami_cleanup(ctx, thread_pool_size):
+    import reconcile.aws_ami_cleanup.integration
+
+    run_integration(reconcile.aws_ami_cleanup.integration, ctx.obj, thread_pool_size)
+
+
 @integration.command(
     short_help="Generate AWS ECR image pull secrets and store them in Vault."
 )

reconcile/gql_definitions/aws_ami_cleanup/asg_namespaces.py

@@ -0,0 +1,132 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query ASGNamespaces {
+  namespaces: namespaces_v1 {
+    name
+    externalResources {
+      provider
+      provisioner {
+        name
+      }
+      ... on NamespaceTerraformProviderResourceAWS_v1 {
+        resources {
+            provider
+            ... on NamespaceTerraformResourceASG_v1 {
+                identifier
+                image {
+                    provider
+                    ... on ASGImageGit_v1 {
+                       tag_name
+                       url
+                       ref
+                    }
+                    ... on ASGImageStatic_v1 {
+                        tag_name
+                        value
+                    }
+                }
+            }
+        }
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union = True
+        extra = Extra.forbid
+
+
+class ExternalResourcesProvisionerV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class NamespaceExternalResourceV1(ConfiguredBaseModel):
+    provider: str = Field(..., alias="provider")
+    provisioner: ExternalResourcesProvisionerV1 = Field(..., alias="provisioner")
+
+
+class NamespaceTerraformResourceAWSV1(ConfiguredBaseModel):
+    provider: str = Field(..., alias="provider")
+
+
+class ASGImageV1(ConfiguredBaseModel):
+    provider: str = Field(..., alias="provider")
+
+
+class ASGImageGitV1(ASGImageV1):
+    tag_name: str = Field(..., alias="tag_name")
+    url: str = Field(..., alias="url")
+    ref: str = Field(..., alias="ref")
+
+
+class ASGImageStaticV1(ASGImageV1):
+    tag_name: str = Field(..., alias="tag_name")
+    value: str = Field(..., alias="value")
+
+
+class NamespaceTerraformResourceASGV1(NamespaceTerraformResourceAWSV1):
+    identifier: str = Field(..., alias="identifier")
+    image: list[Union[ASGImageGitV1, ASGImageStaticV1, ASGImageV1]] = Field(
+        ..., alias="image"
+    )
+
+
+class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):
+    resources: list[
+        Union[NamespaceTerraformResourceASGV1, NamespaceTerraformResourceAWSV1]
+    ] = Field(..., alias="resources")
+
+
+class NamespaceV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    external_resources: Optional[
+        list[
+            Union[NamespaceTerraformProviderResourceAWSV1, NamespaceExternalResourceV1]
+        ]
+    ] = Field(..., alias="externalResources")
+
+
+class ASGNamespacesQueryData(ConfiguredBaseModel):
+    namespaces: Optional[list[NamespaceV1]] = Field(..., alias="namespaces")
+
+
+def query(query_func: Callable, **kwargs: Any) -> ASGNamespacesQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        ASGNamespacesQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return ASGNamespacesQueryData(**raw_data)

reconcile/queries.py

@@ -535,6 +535,16 @@ AWS_ACCOUNTS_QUERY = """
       }
     }
     {% endif %}
+    {% if cleanup %}
+    cleanup {
+      provider
+      ... on AWSAccountCleanupOptionAMI_v1 {
+        regex
+        age
+        region
+      }
+    }
+    {% endif %}
   }
 }
 """
@@ -547,6 +557,7 @@ def get_aws_accounts(
     sharing=False,
     terraform_state=False,
     ecrs=True,
+    cleanup=False,
 ):
     """Returns all AWS accounts"""
     gqlapi = gql.get_api()
@@ -559,6 +570,7 @@ def get_aws_accounts(
         sharing=sharing,
         terraform_state=terraform_state,
         ecrs=ecrs,
+        cleanup=cleanup,
     )
     return gqlapi.query(query)["accounts"]
 

reconcile/test/test_aws_ami_cleanup.py

@@ -0,0 +1,296 @@
+# pylint does not consider frozen BaseModels as hashable and then complains that they cannot
+# be members of a set.
+# pylint: disable=unhashable-member
+
+from collections.abc import Generator
+from datetime import (
+    datetime,
+    timedelta,
+)
+from typing import (
+    TYPE_CHECKING,
+    Any,
+)
+from unittest.mock import MagicMock
+
+import boto3
+import pytest
+from moto import mock_ec2
+from pytest_mock import MockerFixture
+
+from reconcile.aws_ami_cleanup.integration import (
+    AIAmi,
+    AmiTag,
+    AWSAmi,
+    CannotCompareTagsError,
+    check_aws_ami_in_use,
+    get_app_interface_amis,
+    get_aws_amis,
+)
+from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
+    ASGNamespacesQueryData,
+)
+from reconcile.test.fixtures import Fixtures
+from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
+
+if TYPE_CHECKING:
+    from mypy_boto3_ec2 import EC2Client
+    from mypy_boto3_ec2.type_defs import CreateImageResultTypeDef
+else:
+    EC2Client = object
+    CreateImageResultTypeDef = dict
+
+MOTO_DEFAULT_ACCOUNT = "123456789012"
+
+
+@pytest.fixture
+def accounts() -> list[dict[str, Any]]:
+    return [
+        {
+            "name": "some-account",
+            "automationToken": {
+                "path": "path",
+            },
+            "resourcesDefaultRegion": "default-region",
+        }
+    ]
+
+
+@pytest.fixture
+def aws_api(accounts: list[dict[str, Any]], mocker: MockerFixture) -> AWSApi:
+    mock_secret_reader = mocker.patch(
+        "reconcile.utils.aws_api.SecretReader", autospec=True
+    )
+    mock_secret_reader.return_value.read_all.return_value = {
+        "aws_access_key_id": "key_id",
+        "aws_secret_access_key": "access_key",
+        "region": "tf_state_bucket_region",
+    }
+    return AWSApi(1, accounts, init_users=False)
+
+
+@pytest.fixture
+def ec2_client() -> Generator[EC2Client, None, None]:
+    with mock_ec2():
+        yield boto3.client("ec2", region_name="us-east-1")
+
+
+@pytest.fixture
+def rhel_image(ec2_client: EC2Client) -> CreateImageResultTypeDef:
+    # RHEL7 ami from moto/ec2/resources/amis.json
+    reservation = ec2_client.run_instances(
+        ImageId="ami-bb9a6bc2", MinCount=1, MaxCount=1
+    )
+    instance_id = reservation["Instances"][0]["InstanceId"]
+
+    return ec2_client.create_image(
+        InstanceId=instance_id,
+        Name="ci-int-jenkins-worker-rhel7-sha-123456",
+        TagSpecifications=[
+            {
+                "ResourceType": "image",
+                "Tags": [
+                    {"Key": "infra_commit", "Value": "sha-123456"},
+                    {"Key": "type", "Value": "ci-int-jenkins-worker-rhel7"},
+                ],
+            },
+        ],
+    )
+
+
+@pytest.fixture
+def suse_image(ec2_client: EC2Client) -> CreateImageResultTypeDef:
+    # SUSE AMI from moto/ec2/resources/amis.json
+    reservation = ec2_client.run_instances(
+        ImageId="ami-35e92e4c", MinCount=1, MaxCount=1
+    )
+    instance_id = reservation["Instances"][0]["InstanceId"]
+
+    return ec2_client.create_image(
+        InstanceId=instance_id,
+        Name="ci-int-jenkins-worker-suse12-sha-789012",
+        TagSpecifications=[
+            {
+                "ResourceType": "image",
+                "Tags": [
+                    {"Key": "infra_commit", "Value": "sha-789012"},
+                    {"Key": "arch", "Value": "ci-int-jenkins-worker-suse12"},
+                ],
+            },
+        ],
+    )
+
+
+@pytest.fixture
+def ai_amis_fxt() -> list[AIAmi]:
+    return [
+        AIAmi(
+            identifier="ci-int-jenkins-worker-app-sre",
+            tags={
+                AmiTag(key="type", value="ci-int-jenkins-worker-app-sre"),
+                AmiTag(
+                    key="infra_commit",
+                    value="sha-0123",
+                ),
+            },
+        ),
+        AIAmi(
+            identifier="ci-int-jenkins-worker-app-interface",
+            tags={
+                AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
+                AmiTag(
+                    key="infra_commit",
+                    value="sha-4567",
+                ),
+            },
+        ),
+    ]
+
+
+def test_get_aws_amis_success(
+    ec2_client: EC2Client,
+    aws_api: AWSApi,
+    rhel_image: CreateImageResultTypeDef,
+    suse_image: CreateImageResultTypeDef,
+) -> None:
+    utc_now = datetime.utcnow() + timedelta(seconds=60)
+    amis = get_aws_amis(
+        aws_api=aws_api,
+        ec2_client=ec2_client,
+        owner=MOTO_DEFAULT_ACCOUNT,
+        regex="ci-int-jenkins-worker-rhel7.*",
+        age_in_seconds=30,
+        utc_now=utc_now,
+        region="us-east-1",
+    )
+
+    assert len(amis) == 1
+    assert amis[0].image_id == rhel_image["ImageId"]
+
+
+def test_get_aws_amis_unmatched_regex(
+    ec2_client: EC2Client,
+    aws_api: AWSApi,
+    rhel_image: CreateImageResultTypeDef,
+    suse_image: CreateImageResultTypeDef,
+) -> None:
+    utc_now = datetime.utcnow() + timedelta(seconds=60)
+    amis = get_aws_amis(
+        aws_api=aws_api,
+        ec2_client=ec2_client,
+        owner=MOTO_DEFAULT_ACCOUNT,
+        regex="ci-int-jenkins-worker-centos7.*",
+        age_in_seconds=30,
+        utc_now=utc_now,
+        region="us-east-1",
+    )
+
+    assert len(amis) == 0
+
+
+def test_get_aws_amis_different_account(
+    ec2_client: EC2Client,
+    aws_api: AWSApi,
+    rhel_image: CreateImageResultTypeDef,
+    suse_image: CreateImageResultTypeDef,
+) -> None:
+    utc_now = datetime.utcnow() + timedelta(seconds=60)
+    amis = get_aws_amis(
+        aws_api=aws_api,
+        ec2_client=ec2_client,
+        owner="789123456789",
+        regex="ci-int-jenkins-worker-rhel7.*",
+        age_in_seconds=30,
+        utc_now=utc_now,
+        region="us-east-1",
+    )
+
+    assert len(amis) == 0
+
+
+def test_get_aws_amis_too_young(
+    ec2_client: EC2Client,
+    aws_api: AWSApi,
+    rhel_image: CreateImageResultTypeDef,
+    suse_image: CreateImageResultTypeDef,
+) -> None:
+    utc_now = datetime.utcnow() + timedelta(seconds=60)
+    amis = get_aws_amis(
+        aws_api=aws_api,
+        ec2_client=ec2_client,
+        owner=MOTO_DEFAULT_ACCOUNT,
+        regex="ci-int-jenkins-worker-rhel7.*",
+        age_in_seconds=90,
+        utc_now=utc_now,
+        region="us-east-1",
+    )
+
+    assert len(amis) == 0
+
+
+def test_get_app_interface_amis(ai_amis_fxt: list[AIAmi]) -> None:
+    fixture = Fixtures("aws_ami_cleanup").get_anymarkup("namespaces.yaml")
+    namespaces = ASGNamespacesQueryData(**fixture).namespaces
+    ts = MagicMock(spec=Terrascript)
+    ts.get_commit_sha.side_effect = ["sha-0123", "sha-4567"]
+
+    app_interface_amis = get_app_interface_amis(namespaces, ts)
+    assert app_interface_amis[0].identifier == ai_amis_fxt[0].identifier
+    assert app_interface_amis[0].tags == ai_amis_fxt[0].tags
+    assert app_interface_amis[1].identifier == ai_amis_fxt[1].identifier
+    assert app_interface_amis[1].tags == ai_amis_fxt[1].tags
+
+
+def test_check_aws_ami_in_use(ai_amis_fxt: list[AIAmi]) -> None:
+    utc_now = datetime.utcnow()
+    aws_ami = AWSAmi(
+        name="ci-int-jenkins-worker-app-sre-sha-0123",
+        image_id="ami-123456",
+        creation_date=utc_now,
+        tags={
+            AmiTag(key="infra_commit", value="sha-0123"),
+            AmiTag(key="type", value="ci-int-jenkins-worker-app-sre"),
+        },
+    )
+    assert check_aws_ami_in_use(aws_ami, ai_amis_fxt) == "ci-int-jenkins-worker-app-sre"
+
+    aws_ami = AWSAmi(
+        name="ci-int-jenkins-worker-app-interface-sha-4567",
+        image_id="ami-823445",
+        creation_date=utc_now,
+        tags={
+            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
+            AmiTag(key="infra_commit", value="sha-4567"),
+        },
+    )
+    assert (
+        check_aws_ami_in_use(aws_ami, ai_amis_fxt)
+        == "ci-int-jenkins-worker-app-interface"
+    )
+
+    aws_ami = AWSAmi(
+        name="ci-int-jenkins-worker-app-interface-a-different-sha",
+        image_id="ami-823445",
+        creation_date=utc_now,
+        tags={
+            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
+            AmiTag(key="infra_commit", value="a-different-sha"),
+        },
+    )
+
+    assert not check_aws_ami_in_use(aws_ami, ai_amis_fxt)
+
+    aws_ami = AWSAmi(
+        name="ci-int-jenkins-worker-app-interface-a-weird-one",
+        image_id="ami-823445",
+        creation_date=utc_now,
+        tags={
+            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
+        },
+    )
+
+    with pytest.raises(CannotCompareTagsError) as excinfo:
+        check_aws_ami_in_use(aws_ami, ai_amis_fxt)
+
+    assert "AI AMI has more tags than" in str(excinfo.value)