qontract-reconcile 0.10.1rc1052__py3-none-any.whl → 0.10.1rc1053__py3-none-any.whl

{qontract_reconcile-0.10.1rc1052.dist-info → qontract_reconcile-0.10.1rc1053.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1052
+Version: 0.10.1rc1053
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1052.dist-info → qontract_reconcile-0.10.1rc1053.dist-info}/RECORD

@@ -145,7 +145,7 @@ reconcile/aws_account_manager/metrics.py,sha256=YB10ea4kIGwJfs5N14RF-RoXPb-QQWaD
 reconcile/aws_account_manager/reconciler.py,sha256=8mwwcWVVNoUzmttzxgnLyePqN823v1t_dQCNCrx1mG0,15035
 reconcile/aws_account_manager/utils.py,sha256=iYPPOtbZ7FiKkz9v5f1YXRIHw5YFOtSavUkF8oMwfJY,1439
 reconcile/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aws_ami_cleanup/integration.py,sha256=JkKOcH_7uKBmjdRK1UUoWseUik3kBIjwdtRdPMUEkBY,10086
+reconcile/aws_ami_cleanup/integration.py,sha256=KG7g9NpbKmoaveDD3oi9SinqUE29NaM-4lGo-6YuHlM,9302
 reconcile/aws_cloudwatch_log_retention/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_cloudwatch_log_retention/integration.py,sha256=0UcSZIrGvnGY4m9fj87oejIolIP_qTxtJInpmW9jrQ0,7772
 reconcile/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -222,7 +222,7 @@ reconcile/gql_definitions/app_interface_metrics_exporter/onboarding_status.py,sh
 reconcile/gql_definitions/aws_account_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/aws_account_manager/aws_accounts.py,sha256=JdqtE3gMpeodymPJST-aFVkYP_MO--_CcwjF070R5Cs,4883
 reconcile/gql_definitions/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/aws_ami_cleanup/asg_namespaces.py,sha256=OJmeTu7uirLGAysZ3IQTtRXqMyL8noi_QZxPuWYxxmI,3678
+reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py,sha256=jIgOa888MYLLvVsn1ir3nbkhWLG5T6dBg7oDnp1q8BI,4108
 reconcile/gql_definitions/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/aws_saml_idp/aws_accounts.py,sha256=pR9Qm6P9Roe4OJaDXvfm8AcfkSSAtriQdlwLwW7UdUU,2666
 reconcile/gql_definitions/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -500,7 +500,7 @@ reconcile/test/test_acs_policies.py,sha256=8pwnXpAO-0OI-6oubjf_oPPlpZjVldeZfJJ9u
 reconcile/test/test_acs_rbac.py,sha256=lvNd8GY0-GHzcOdOn13QWdrqbBXXKzNT7EEDHNH7cjM,28272
 reconcile/test/test_aggregated_list.py,sha256=iiWitQuNYC58aimWaiBoE4NROHjr1NCgQ91MnHEG_Ro,6412
 reconcile/test/test_amtool.py,sha256=vxRhGieeydMBOb9UI2ziMHjJa8puMeGNsUhGhy-yMnk,1032
-reconcile/test/test_aws_ami_cleanup.py,sha256=cHumkZD33vIIblNbrMKNCva-WeE3sMv8kiFA0A96wwk,8733
+reconcile/test/test_aws_ami_cleanup.py,sha256=s9E82ENpKrzsy9_KTj0tbQN2MVXeacq-_Ztw7FWbq7Q,6603
 reconcile/test/test_aws_ami_share.py,sha256=eSITdDoXs8mMY7P2lFxAX2DA0sJ9RW6D1tG8Rek0gLE,1981
 reconcile/test/test_aws_cloudwatch_log_retention.py,sha256=fOUq_vJldGPjXKBdRQsSv4zFz_qenVtRKJFKpbz8BlA,13712
 reconcile/test/test_aws_iam_keys.py,sha256=MfE9EvItyPNPAl5QaLlJFUvvrZFiar518TM2wWNjJn4,1829
@@ -867,8 +867,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1052.dist-info/METADATA,sha256=vApqFMxNENnXiau-v2JHl7G_NMa4KoDW4q9N9CAFcvM,2213
-qontract_reconcile-0.10.1rc1052.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1052.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1052.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1052.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1053.dist-info/METADATA,sha256=DIX4cB4NDK1taSiJkGbWr1jmAZIQpsQHp_6xGaNjKQI,2213
+qontract_reconcile-0.10.1rc1053.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1053.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1053.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1053.dist-info/RECORD,,

reconcile/aws_ami_cleanup/integration.py

@@ -1,9 +1,8 @@
 import logging
 import re
-import sys
+from collections import defaultdict
 from collections.abc import (
     Callable,
-    Mapping,
 )
 from datetime import (
     datetime,
@@ -11,27 +10,20 @@ from datetime import (
 )
 from typing import (
     TYPE_CHECKING,
-    Any,
 )
 
 from botocore.exceptions import ClientError
 from pydantic import (
     BaseModel,
-    Field,
 )
 
-from reconcile import queries
-from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
-    ASGImageGitV1,
-    ASGImageStaticV1,
-    NamespaceTerraformProviderResourceAWSV1,
-    NamespaceTerraformResourceASGV1,
-    NamespaceV1,
+from reconcile.gql_definitions.aws_ami_cleanup.aws_accounts import (
+    AWSAccountCleanupOptionAMIV1,
+    AWSAccountSharingOptionAMIV1,
 )
-from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
-    query as query_asg_namespaces,
+from reconcile.gql_definitions.aws_ami_cleanup.aws_accounts import (
+    query as aws_accounts_query,
 )
-from reconcile.status import ExitCodes
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
@@ -40,7 +32,6 @@ from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.defer import defer
 from reconcile.utils.parse_dhms_duration import dhms_to_seconds
 from reconcile.utils.secret_reader import create_secret_reader
-from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
 
 if TYPE_CHECKING:
     from mypy_boto3_ec2 import EC2Client
@@ -51,23 +42,9 @@ QONTRACT_INTEGRATION = "aws_ami_cleanup"
 MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
 
 
-class CannotCompareTagsError(Exception):
-    pass
-
-
-class AmiTag(BaseModel):
-    key: str = Field(alias="Key")
-    value: str = Field(alias="Value")
-
-    class Config:
-        allow_population_by_field_name = True
-        frozen = True
-
-
 class AWSAmi(BaseModel):
     name: str
     image_id: str
-    tags: set[AmiTag]
     creation_date: datetime
     snapshot_ids: list[str]
 
@@ -75,237 +52,214 @@ class AWSAmi(BaseModel):
         frozen = True
 
 
-class AIAmi(BaseModel):
-    identifier: str
-    tags: set[AmiTag]
+def get_aws_amis_from_launch_templates(ec2_client: EC2Client) -> set[str]:
+    amis = set()
 
-    class Config:
-        frozen = True
+    paginator = ec2_client.get_paginator("describe_launch_templates")
+    for page in paginator.paginate():
+        for launch_template in page.get("LaunchTemplates", []):
+            if launch_template_id := launch_template.get("LaunchTemplateId"):
+                launch_template_versions = ec2_client.describe_launch_template_versions(
+                    LaunchTemplateId=launch_template_id,
+                    Versions=[str(launch_template.get("LatestVersionNumber"))],
+                ).get("LaunchTemplateVersions", [])
+
+                if launch_template_versions:
+                    if (
+                        ami_id := launch_template_versions[0]
+                        .get("LaunchTemplateData", {})
+                        .get("ImageId")
+                    ):
+                        amis.add(ami_id)
+
+    return amis
 
 
 def get_aws_amis(
-    aws_api: AWSApi,
     ec2_client: EC2Client,
     owner: str,
     regex: str,
     age_in_seconds: int,
     utc_now: datetime,
-    region: str,
 ) -> list[AWSAmi]:
     """Get amis that match regex older than given age"""
 
-    images = aws_api.paginate(
-        ec2_client, "describe_images", "Images", {"Owners": [owner]}
-    )
-
     pattern = re.compile(regex)
+    paginator = ec2_client.get_paginator("describe_images")
     results = []
-    for i in images:
-        if not re.search(pattern, i["Name"]):
-            continue
+    for page in paginator.paginate(Owners=[owner]):
+        for image in page.get("Images", []):
+            if not re.search(pattern, image["Name"]):
+                continue
 
-        creation_date = datetime.strptime(i["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ")
-        current_delta = utc_now - creation_date
-        delete_delta = timedelta(seconds=age_in_seconds)
+            creation_date = datetime.strptime(
+                image["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+            )
+            current_delta = utc_now - creation_date
+            delete_delta = timedelta(seconds=age_in_seconds)
 
-        if current_delta < delete_delta:
-            continue
+            if current_delta < delete_delta:
+                continue
 
-        # We have nothing to do with untagged AMIs since we will need tags to verify if AMI are
-        # in use or not.
-        if not i.get("Tags"):
-            continue
+            snapshot_ids = []
+            for bdv in image["BlockDeviceMappings"]:
+                ebs = bdv.get("Ebs")
+                if not ebs:
+                    continue
 
-        snapshot_ids = []
-        for bdv in i["BlockDeviceMappings"]:
-            ebs = bdv.get("Ebs")
-            if not ebs:
-                continue
+                if sid := ebs.get("SnapshotId"):
+                    snapshot_ids.append(sid)
 
-            if sid := ebs.get("SnapshotId"):
-                snapshot_ids.append(sid)
-
-        tags = {AmiTag(**tag) for tag in i.get("Tags")}
-        results.append(
-            AWSAmi(
-                name=i["Name"],
-                image_id=i["ImageId"],
-                tags=tags,
-                creation_date=creation_date,
-                snapshot_ids=snapshot_ids,
+            results.append(
+                AWSAmi(
+                    name=image["Name"],
+                    image_id=image["ImageId"],
+                    creation_date=creation_date,
+                    snapshot_ids=snapshot_ids,
+                )
             )
-        )
 
     return results
 
 
 def get_region(
-    cleanup: Mapping[str, Any],
-    account: Mapping[str, Any],
+    config_region: str | None,
+    account_name: str,
+    resources_default_region: str,
+    supported_deployment_regions: list[str] | None,
 ) -> str:
     """Defines the region to search for AMIs."""
-    region = cleanup.get("region") or account["resourcesDefaultRegion"]
-    if region not in account["supportedDeploymentRegions"]:
-        raise ValueError(f"region {region} is not supported in {account['name']}")
+    region = config_region or resources_default_region
+    if region not in (supported_deployment_regions or []):
+        raise ValueError(f"region {region} is not supported in {account_name}")
 
     return region
 
 
-def get_app_interface_amis(
-    namespaces: list[NamespaceV1] | None, ts: Terrascript
-) -> list[AIAmi]:
-    """Returns all the ami referenced in ASGs in app-interface."""
-    app_interface_amis = []
-    for n in namespaces or []:
-        for er in n.external_resources or []:
-            if not isinstance(er, NamespaceTerraformProviderResourceAWSV1):
-                continue
-
-            for r in er.resources:
-                if not isinstance(r, NamespaceTerraformResourceASGV1):
-                    continue
-
-                tags = set()
-                for i in r.image:
-                    if isinstance(i, ASGImageGitV1):
-                        tags.add(
-                            AmiTag(
-                                key=i.tag_name,
-                                value=ts.get_commit_sha(i.dict(by_alias=True)),
-                            )
-                        )
-                    elif isinstance(i, ASGImageStaticV1):
-                        tags.add(AmiTag(key=i.tag_name, value=i.value))
-
-                app_interface_amis.append(AIAmi(identifier=r.identifier, tags=tags))
-
-    return app_interface_amis
-
-
-def check_aws_ami_in_use(
-    aws_ami: AWSAmi, app_interface_amis: list[AIAmi]
-) -> str | None:
-    """Verifies if the given AWS ami is in use in a defined app-interface ASG."""
-    for ai_ami in app_interface_amis:
-        # This can happen if the ASG init template has changed over the time. We don't have a way
-        # to properly delete these automatically since we cannot assure they are not in use.
-        # The integration will fail in this case and these amis will need to be handled manually.
-        if len(ai_ami.tags) > len(aws_ami.tags):
-            raise CannotCompareTagsError(
-                f"{ai_ami.identifier} AI AMI has more tags than {aws_ami.image_id} AWS AMI"
-            )
-
-        if ai_ami.tags.issubset(aws_ami.tags):
-            return ai_ami.identifier
-
-    return None
-
-
 @defer
 def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) -> None:
-    exit_code = ExitCodes.SUCCESS
-
-    # We still use here a non-typed query; accounts is passed to AWSApi and Terrascript classes
-    # which contain a vast amount of magic based on keys from that dict. Since this integration
-    # cannot still be properly monitored (see https://issues.redhat.com/browse/APPSRE-7674),
-    # it's easy that it breaks without being noticed. Once it is properly monitored, this should
-    # be moved to a typed query.
-    query_data = queries.get_aws_accounts(terraform_state=True, cleanup=True)
+    utc_now = datetime.utcnow()
+    gqlapi = gql.get_api()
+    aws_accounts = aws_accounts_query(gqlapi.query).accounts
+
+    # Get accounts that have cleanup configured
     cleanup_accounts = []
-    for data in query_data:
-        cleanups = data.get("cleanup")
-        if not cleanups:
+    for account in aws_accounts or []:
+        if not account.cleanup:
             continue
         is_ami_related = False
-        for cleanup in cleanups:
-            is_ami_related |= cleanup.get("provider") == "ami"
+        for cleanup in account.cleanup:
+            is_ami_related |= cleanup.provider == "ami"
         if not is_ami_related:
             continue
-        cleanup_accounts.append(data)
+        cleanup_accounts.append(account)
 
-    vault_settings = get_app_interface_vault_settings()
+    # Build a dict with all accounts that are used to share amis with. Together with
+    # the cleanup account we will look into the account's launch templates in order to
+    # find the AMIs that are currently being used. We will make sure that those are not
+    # deleted even if they have expired.
+    ami_accounts = defaultdict(set)
+    for account in cleanup_accounts:
+        ami_accounts[account.name].add(account.resources_default_region)
 
-    ts = Terrascript(
-        QONTRACT_INTEGRATION,
-        "",
-        thread_pool_size,
-        cleanup_accounts,
-        settings=vault_settings.dict(by_alias=True),
-    )
+        for sharing_config in account.sharing or []:
+            if not isinstance(sharing_config, AWSAccountSharingOptionAMIV1):
+                continue
 
-    gqlapi = gql.get_api()
-    namespaces = query_asg_namespaces(query_func=gqlapi.query).namespaces or []
-    app_interface_amis = get_app_interface_amis(namespaces, ts)
+            region = get_region(
+                config_region=sharing_config.region,
+                account_name=sharing_config.account.name,
+                resources_default_region=sharing_config.account.resources_default_region,
+                supported_deployment_regions=sharing_config.account.supported_deployment_regions,
+            )
 
-    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
-    aws_api = AWSApi(1, cleanup_accounts, secret_reader=secret_reader, init_users=False)
+            ami_accounts[sharing_config.account.name].add(
+                sharing_config.account.resources_default_region
+            )
+
+    # Build AWSApi object. We will use all those accounts listed in ami_accounts since
+    # we will also need to look for used AMIs.
+    accounts_dicted = [
+        account.dict(by_alias=True)
+        for account in aws_accounts or []
+        if account.name in ami_accounts
+    ]
+    secret_reader = create_secret_reader(
+        use_vault=get_app_interface_vault_settings().vault
+    )
+    aws_api = AWSApi(1, accounts_dicted, secret_reader=secret_reader, init_users=False)
     if defer:  # defer is provided by the method decorator; this makes just mypy happy.
         defer(aws_api.cleanup)
 
-    utc_now = datetime.utcnow()
+    # Get all AMIs used
+    amis_used_in_launch_templates = set()
+    for account_name, regions in ami_accounts.items():
+        for region in regions:
+            session = aws_api.get_session(account_name)
+            ec2_client = aws_api.get_session_client(session, "ec2", region)
+            launch_template_amis = get_aws_amis_from_launch_templates(
+                ec2_client=ec2_client
+            )
+            if launch_template_amis:
+                amis_used_in_launch_templates.update(launch_template_amis)
+
+    # The action
     for account in cleanup_accounts:
-        for cleanup in account["cleanup"]:
-            if cleanup["provider"] != "ami":
+        for cleanup_config in account.cleanup or []:
+            if not isinstance(cleanup_config, AWSAccountCleanupOptionAMIV1):
                 continue
 
-            region = get_region(cleanup, account)
-            regex = cleanup["regex"]
-            age_in_seconds = dhms_to_seconds(cleanup["age"])
+            region = get_region(
+                config_region=cleanup_config.region,
+                account_name=account.name,
+                resources_default_region=account.resources_default_region,
+                supported_deployment_regions=account.supported_deployment_regions,
+            )
+            age_in_seconds = dhms_to_seconds(cleanup_config.age)
 
-            session = aws_api.get_session(account["name"])
+            session = aws_api.get_session(account.name)
             ec2_client = aws_api.get_session_client(session, "ec2", region)
 
-            amis = get_aws_amis(
-                aws_api=aws_api,
+            aws_amis = get_aws_amis(
                 ec2_client=ec2_client,
-                owner=account["uid"],
-                regex=regex,
+                owner=account.uid,
+                regex=cleanup_config.regex,
                 age_in_seconds=age_in_seconds,
                 utc_now=utc_now,
-                region=region,
             )
 
-            for aws_ami in amis:
-                try:
-                    if identifier := check_aws_ami_in_use(aws_ami, app_interface_amis):
-                        logging.info(
-                            "Discarding ami %s with id %s as it is used in app-interface in %s",
-                            aws_ami.name,
-                            aws_ami.image_id,
-                            identifier,
-                        )
-                        continue
-                except CannotCompareTagsError as e:
-                    logging.error(e)
-                    if not dry_run:
-                        exit_code = ExitCodes.ERROR
+            for ami in aws_amis:
+                if ami.image_id in amis_used_in_launch_templates:
+                    logging.info(
+                        "Discarding AMI %s with id %s as it is still in use.",
+                        ami.name,
+                        ami.image_id,
+                    )
                     continue
 
                 logging.info(
                     "Deregistering image %s with id %s created in %s",
-                    aws_ami.name,
-                    aws_ami.image_id,
-                    aws_ami.creation_date,
+                    ami.name,
+                    ami.image_id,
+                    ami.creation_date,
                 )
 
                 try:
-                    ec2_client.deregister_image(
-                        ImageId=aws_ami.image_id, DryRun=dry_run
-                    )
+                    ec2_client.deregister_image(ImageId=ami.image_id, DryRun=dry_run)
                 except ClientError as e:
                     if "DryRunOperation" in str(e):
                         logging.info(e)
                     else:
                         raise
 
-                if not aws_ami.snapshot_ids:
+                if not ami.snapshot_ids:
                     continue
 
-                for snapshot_id in aws_ami.snapshot_ids:
+                for snapshot_id in ami.snapshot_ids:
                     logging.info(
                         "Deleting associated snapshot %s from image %s",
                         snapshot_id,
-                        aws_ami.image_id,
+                        ami.image_id,
                     )
 
                     try:
@@ -317,5 +271,3 @@ def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) ->
                             logging.info(e)
                         else:
                             raise
-
-    sys.exit(exit_code)

reconcile/gql_definitions/aws_ami_cleanup/aws_accounts.py

@@ -0,0 +1,161 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.aws_account_common import AWSAccountCommon
+from reconcile.gql_definitions.fragments.terraform_state import TerraformState
+
+
+DEFINITION = """
+fragment AWSAccountCommon on AWSAccount_v1 {
+  path
+  name
+  description
+  uid
+  terraformUsername
+  consoleUrl
+  resourcesDefaultRegion
+  supportedDeploymentRegions
+  providerVersion
+  accountOwners {
+    name
+    email
+  }
+  automationToken {
+    ... VaultSecret
+  }
+  garbageCollection
+  enableDeletion
+  deletionApprovals {
+    type
+    name
+    expiration
+  }
+  disable {
+    integrations
+  }
+  deleteKeys
+  premiumSupport
+  partition
+}
+
+fragment TerraformState on TerraformStateAWS_v1 {
+  provider
+  bucket
+  region
+  integrations {
+    key
+    integration
+  }
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query AWSAccountsAmiCleanup {
+  accounts: awsaccounts_v1 {
+    ...AWSAccountCommon
+    terraformState {
+      ...TerraformState
+    }
+    cleanup {
+      provider
+      ... on AWSAccountCleanupOptionAMI_v1 {
+        regex
+        age
+        region
+      }
+    }
+    sharing {
+      provider
+      account {
+        name
+        supportedDeploymentRegions
+        resourcesDefaultRegion
+      }
+      ... on AWSAccountSharingOptionAMI_v1 {
+        region
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AWSAccountCleanupOptionV1(ConfiguredBaseModel):
+    provider: str = Field(..., alias="provider")
+
+
+class AWSAccountCleanupOptionAMIV1(AWSAccountCleanupOptionV1):
+    regex: str = Field(..., alias="regex")
+    age: str = Field(..., alias="age")
+    region: Optional[str] = Field(..., alias="region")
+
+
+class AWSAccountSharingOptionV1_AWSAccountV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    supported_deployment_regions: Optional[list[str]] = Field(..., alias="supportedDeploymentRegions")
+    resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
+
+
+class AWSAccountSharingOptionV1(ConfiguredBaseModel):
+    provider: str = Field(..., alias="provider")
+    account: AWSAccountSharingOptionV1_AWSAccountV1 = Field(..., alias="account")
+
+
+class AWSAccountSharingOptionAMIV1(AWSAccountSharingOptionV1):
+    region: Optional[str] = Field(..., alias="region")
+
+
+class AWSAccountV1(AWSAccountCommon):
+    terraform_state: Optional[TerraformState] = Field(..., alias="terraformState")
+    cleanup: Optional[list[Union[AWSAccountCleanupOptionAMIV1, AWSAccountCleanupOptionV1]]] = Field(..., alias="cleanup")
+    sharing: Optional[list[Union[AWSAccountSharingOptionAMIV1, AWSAccountSharingOptionV1]]] = Field(..., alias="sharing")
+
+
+class AWSAccountsAmiCleanupQueryData(ConfiguredBaseModel):
+    accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
+
+
+def query(query_func: Callable, **kwargs: Any) -> AWSAccountsAmiCleanupQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        AWSAccountsAmiCleanupQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return AWSAccountsAmiCleanupQueryData(**raw_data)

reconcile/test/test_aws_ami_cleanup.py

@@ -11,35 +11,26 @@ from typing import (
     TYPE_CHECKING,
     Any,
 )
-from unittest.mock import MagicMock
 
 import boto3
 import pytest
 from moto import mock_ec2
-from pytest_mock import MockerFixture
 
 from reconcile.aws_ami_cleanup.integration import (
-    AIAmi,
-    AmiTag,
-    AWSAmi,
-    CannotCompareTagsError,
-    check_aws_ami_in_use,
-    get_app_interface_amis,
     get_aws_amis,
+    get_aws_amis_from_launch_templates,
 )
-from reconcile.gql_definitions.aws_ami_cleanup.asg_namespaces import (
-    ASGNamespacesQueryData,
-)
-from reconcile.test.fixtures import Fixtures
-from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
 
 if TYPE_CHECKING:
     from mypy_boto3_ec2 import EC2Client
-    from mypy_boto3_ec2.type_defs import CreateImageResultTypeDef
+    from mypy_boto3_ec2.type_defs import (
+        CreateImageResultTypeDef,
+        LaunchTemplateVersionTypeDef,
+    )
 else:
     EC2Client = object
     CreateImageResultTypeDef = dict
+    LaunchTemplateVersionTypeDef = dict
 
 MOTO_DEFAULT_ACCOUNT = "123456789012"
 
@@ -57,19 +48,6 @@ def accounts() -> list[dict[str, Any]]:
     ]
 
 
-@pytest.fixture
-def aws_api(accounts: list[dict[str, Any]], mocker: MockerFixture) -> AWSApi:
-    mock_secret_reader = mocker.patch(
-        "reconcile.utils.aws_api.SecretReader", autospec=True
-    )
-    mock_secret_reader.return_value.read_all.return_value = {
-        "aws_access_key_id": "key_id",
-        "aws_secret_access_key": "access_key",
-        "region": "tf_state_bucket_region",
-    }
-    return AWSApi(1, accounts, init_users=False)
-
-
 @pytest.fixture
 def ec2_client() -> Generator[EC2Client, None, None]:
     with mock_ec2():
@@ -123,46 +101,53 @@ def suse_image(ec2_client: EC2Client) -> CreateImageResultTypeDef:
 
 
 @pytest.fixture
-def ai_amis_fxt() -> list[AIAmi]:
-    return [
-        AIAmi(
-            identifier="ci-int-jenkins-worker-app-sre",
-            tags={
-                AmiTag(key="type", value="ci-int-jenkins-worker-app-sre"),
-                AmiTag(
-                    key="infra_commit",
-                    value="sha-0123",
-                ),
-            },
-        ),
-        AIAmi(
-            identifier="ci-int-jenkins-worker-app-interface",
-            tags={
-                AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
-                AmiTag(
-                    key="infra_commit",
-                    value="sha-4567",
-                ),
-            },
-        ),
-    ]
+def ubuntu_launch_template(ec2_client: EC2Client) -> LaunchTemplateVersionTypeDef:
+    # Ubuntu AMI from moto/ec2/resources/amis.json
+    response = ec2_client.create_launch_template(
+        LaunchTemplateName="Ubuntu",
+        LaunchTemplateData={
+            "ImageId": "ami-1e749f67",  # Canonical, Ubuntu, 14.04 LTS
+            "InstanceType": "t3.micro",
+            "SecurityGroupIds": ["sg-12345678"],
+        },
+    )
+
+    return ec2_client.describe_launch_template_versions(
+        LaunchTemplateId=response["LaunchTemplate"]["LaunchTemplateId"],
+        Versions=["1"],
+    )["LaunchTemplateVersions"][0]
+
+
+@pytest.fixture
+def suse_launch_template(ec2_client: EC2Client) -> LaunchTemplateVersionTypeDef:
+    # SUSE AMI from moto/ec2/resources/amis.json
+    response = ec2_client.create_launch_template(
+        LaunchTemplateName="SUSE",
+        LaunchTemplateData={
+            "ImageId": "ami-35e92e4c",  # SUSE Linux Enterprise Server 12 SP3
+            "InstanceType": "t3.micro",
+            "SecurityGroupIds": ["sg-12345678"],
+        },
+    )
+
+    return ec2_client.describe_launch_template_versions(
+        LaunchTemplateId=response["LaunchTemplate"]["LaunchTemplateId"],
+        Versions=["1"],
+    )["LaunchTemplateVersions"][0]
 
 
 def test_get_aws_amis_success(
     ec2_client: EC2Client,
-    aws_api: AWSApi,
     rhel_image: CreateImageResultTypeDef,
     suse_image: CreateImageResultTypeDef,
 ) -> None:
     utc_now = datetime.utcnow() + timedelta(seconds=60)
     amis = get_aws_amis(
-        aws_api=aws_api,
         ec2_client=ec2_client,
         owner=MOTO_DEFAULT_ACCOUNT,
         regex="ci-int-jenkins-worker-rhel7.*",
         age_in_seconds=30,
         utc_now=utc_now,
-        region="us-east-1",
     )
 
     assert len(amis) == 1
@@ -171,19 +156,16 @@ def test_get_aws_amis_success(
 
 def test_get_aws_amis_unmatched_regex(
     ec2_client: EC2Client,
-    aws_api: AWSApi,
     rhel_image: CreateImageResultTypeDef,
     suse_image: CreateImageResultTypeDef,
 ) -> None:
     utc_now = datetime.utcnow() + timedelta(seconds=60)
     amis = get_aws_amis(
-        aws_api=aws_api,
         ec2_client=ec2_client,
         owner=MOTO_DEFAULT_ACCOUNT,
         regex="ci-int-jenkins-worker-centos7.*",
         age_in_seconds=30,
         utc_now=utc_now,
-        region="us-east-1",
     )
 
     assert len(amis) == 0
@@ -191,19 +173,16 @@ def test_get_aws_amis_unmatched_regex(
 
 def test_get_aws_amis_different_account(
     ec2_client: EC2Client,
-    aws_api: AWSApi,
     rhel_image: CreateImageResultTypeDef,
     suse_image: CreateImageResultTypeDef,
 ) -> None:
     utc_now = datetime.utcnow() + timedelta(seconds=60)
     amis = get_aws_amis(
-        aws_api=aws_api,
         ec2_client=ec2_client,
         owner="789123456789",
         regex="ci-int-jenkins-worker-rhel7.*",
         age_in_seconds=30,
         utc_now=utc_now,
-        region="us-east-1",
     )
 
     assert len(amis) == 0
@@ -211,90 +190,41 @@ def test_get_aws_amis_different_account(
 
 def test_get_aws_amis_too_young(
     ec2_client: EC2Client,
-    aws_api: AWSApi,
     rhel_image: CreateImageResultTypeDef,
     suse_image: CreateImageResultTypeDef,
 ) -> None:
     utc_now = datetime.utcnow() + timedelta(seconds=60)
     amis = get_aws_amis(
-        aws_api=aws_api,
         ec2_client=ec2_client,
         owner=MOTO_DEFAULT_ACCOUNT,
         regex="ci-int-jenkins-worker-rhel7.*",
         age_in_seconds=90,
         utc_now=utc_now,
-        region="us-east-1",
     )
 
     assert len(amis) == 0
 
 
-def test_get_app_interface_amis(ai_amis_fxt: list[AIAmi]) -> None:
-    fixture = Fixtures("aws_ami_cleanup").get_anymarkup("namespaces.yaml")
-    namespaces = ASGNamespacesQueryData(**fixture).namespaces
-    ts = MagicMock(spec=Terrascript)
-    ts.get_commit_sha.side_effect = ["sha-0123", "sha-4567"]
-
-    app_interface_amis = get_app_interface_amis(namespaces, ts)
-    assert app_interface_amis[0].identifier == ai_amis_fxt[0].identifier
-    assert app_interface_amis[0].tags == ai_amis_fxt[0].tags
-    assert app_interface_amis[1].identifier == ai_amis_fxt[1].identifier
-    assert app_interface_amis[1].tags == ai_amis_fxt[1].tags
-
-
-def test_check_aws_ami_in_use(ai_amis_fxt: list[AIAmi]) -> None:
-    utc_now = datetime.utcnow()
-    aws_ami = AWSAmi(
-        name="ci-int-jenkins-worker-app-sre-sha-0123",
-        image_id="ami-123456",
-        creation_date=utc_now,
-        tags={
-            AmiTag(key="infra_commit", value="sha-0123"),
-            AmiTag(key="type", value="ci-int-jenkins-worker-app-sre"),
-        },
-        snapshot_ids=[],
-    )
-    assert check_aws_ami_in_use(aws_ami, ai_amis_fxt) == "ci-int-jenkins-worker-app-sre"
-
-    aws_ami = AWSAmi(
-        name="ci-int-jenkins-worker-app-interface-sha-4567",
-        image_id="ami-823445",
-        creation_date=utc_now,
-        tags={
-            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
-            AmiTag(key="infra_commit", value="sha-4567"),
-        },
-        snapshot_ids=[],
-    )
-    assert (
-        check_aws_ami_in_use(aws_ami, ai_amis_fxt)
-        == "ci-int-jenkins-worker-app-interface"
-    )
-
-    aws_ami = AWSAmi(
-        name="ci-int-jenkins-worker-app-interface-a-different-sha",
-        image_id="ami-823445",
-        creation_date=utc_now,
-        tags={
-            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
-            AmiTag(key="infra_commit", value="a-different-sha"),
-        },
-        snapshot_ids=[],
-    )
-
-    assert not check_aws_ami_in_use(aws_ami, ai_amis_fxt)
+def test_get_aws_amis_from_launch_templates(
+    ec2_client: EC2Client,
+    ubuntu_launch_template: LaunchTemplateVersionTypeDef,
+    suse_launch_template: LaunchTemplateVersionTypeDef,
+) -> None:
+    amis = get_aws_amis_from_launch_templates(ec2_client)
+    assert amis == {
+        ubuntu_launch_template["LaunchTemplateData"]["ImageId"],
+        suse_launch_template["LaunchTemplateData"]["ImageId"],
+    }
 
-    aws_ami = AWSAmi(
-        name="ci-int-jenkins-worker-app-interface-a-weird-one",
-        image_id="ami-823445",
-        creation_date=utc_now,
-        tags={
-            AmiTag(key="type", value="ci-int-jenkins-worker-app-interface"),
-        },
-        snapshot_ids=[],
+    # create a new ubuntu version
+    new_ami_id = "ami-785db401"  # "Canonical, Ubuntu, 16.04 LTS
+    ec2_client.create_launch_template_version(
+        LaunchTemplateId=ubuntu_launch_template["LaunchTemplateId"],
+        LaunchTemplateData={"ImageId": new_ami_id},
     )
 
-    with pytest.raises(CannotCompareTagsError) as excinfo:
-        check_aws_ami_in_use(aws_ami, ai_amis_fxt)
-
-    assert "AI AMI has more tags than" in str(excinfo.value)
+    amis = get_aws_amis_from_launch_templates(ec2_client)
+    assert amis == {
+        new_ami_id,
+        suse_launch_template["LaunchTemplateData"]["ImageId"],
+    }

reconcile/gql_definitions/aws_ami_cleanup/asg_namespaces.py

@@ -1,124 +0,0 @@
-"""
-Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
-"""
-from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
-from datetime import datetime  # noqa: F401 # pylint: disable=W0611
-from enum import Enum  # noqa: F401 # pylint: disable=W0611
-from typing import (  # noqa: F401 # pylint: disable=W0611
-    Any,
-    Optional,
-    Union,
-)
-
-from pydantic import (  # noqa: F401 # pylint: disable=W0611
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
-
-
-DEFINITION = """
-query ASGNamespaces {
-  namespaces: namespaces_v1 {
-    name
-    externalResources {
-      provider
-      provisioner {
-        name
-      }
-      ... on NamespaceTerraformProviderResourceAWS_v1 {
-        resources {
-            provider
-            ... on NamespaceTerraformResourceASG_v1 {
-                identifier
-                image {
-                    provider
-                    ... on ASGImageGit_v1 {
-                       tag_name
-                       url
-                       ref
-                    }
-                    ... on ASGImageStatic_v1 {
-                        tag_name
-                        value
-                    }
-                }
-            }
-        }
-      }
-    }
-  }
-}
-"""
-
-
-class ConfiguredBaseModel(BaseModel):
-    class Config:
-        smart_union=True
-        extra=Extra.forbid
-
-
-class ExternalResourcesProvisionerV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class NamespaceExternalResourceV1(ConfiguredBaseModel):
-    provider: str = Field(..., alias="provider")
-    provisioner: ExternalResourcesProvisionerV1 = Field(..., alias="provisioner")
-
-
-class NamespaceTerraformResourceAWSV1(ConfiguredBaseModel):
-    provider: str = Field(..., alias="provider")
-
-
-class ASGImageV1(ConfiguredBaseModel):
-    provider: str = Field(..., alias="provider")
-
-
-class ASGImageGitV1(ASGImageV1):
-    tag_name: str = Field(..., alias="tag_name")
-    url: str = Field(..., alias="url")
-    ref: str = Field(..., alias="ref")
-
-
-class ASGImageStaticV1(ASGImageV1):
-    tag_name: str = Field(..., alias="tag_name")
-    value: str = Field(..., alias="value")
-
-
-class NamespaceTerraformResourceASGV1(NamespaceTerraformResourceAWSV1):
-    identifier: str = Field(..., alias="identifier")
-    image: list[Union[ASGImageGitV1, ASGImageStaticV1, ASGImageV1]] = Field(..., alias="image")
-
-
-class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):
-    resources: list[Union[NamespaceTerraformResourceASGV1, NamespaceTerraformResourceAWSV1]] = Field(..., alias="resources")
-
-
-class NamespaceV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    external_resources: Optional[list[Union[NamespaceTerraformProviderResourceAWSV1, NamespaceExternalResourceV1]]] = Field(..., alias="externalResources")
-
-
-class ASGNamespacesQueryData(ConfiguredBaseModel):
-    namespaces: Optional[list[NamespaceV1]] = Field(..., alias="namespaces")
-
-
-def query(query_func: Callable, **kwargs: Any) -> ASGNamespacesQueryData:
-    """
-    This is a convenience function which queries and parses the data into
-    concrete types. It should be compatible with most GQL clients.
-    You do not have to use it to consume the generated data classes.
-    Alternatively, you can also mime and alternate the behavior
-    of this function in the caller.
-
-    Parameters:
-        query_func (Callable): Function which queries your GQL Server
-        kwargs: optional arguments that will be passed to the query function
-
-    Returns:
-        ASGNamespacesQueryData: queried data parsed into generated classes
-    """
-    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return ASGNamespacesQueryData(**raw_data)