qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/aws_ami_cleanup/integration.py

@@ -0,0 +1,273 @@
+import logging
+import re
+from collections import defaultdict
+from collections.abc import (
+    Callable,
+)
+from datetime import (
+    datetime,
+    timedelta,
+)
+from typing import (
+    TYPE_CHECKING,
+)
+
+from botocore.exceptions import ClientError
+from pydantic import (
+    BaseModel,
+)
+
+from reconcile.gql_definitions.aws_ami_cleanup.aws_accounts import (
+    AWSAccountCleanupOptionAMIV1,
+    AWSAccountSharingOptionAMIV1,
+)
+from reconcile.gql_definitions.aws_ami_cleanup.aws_accounts import (
+    query as aws_accounts_query,
+)
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.utils import gql
+from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.defer import defer
+from reconcile.utils.parse_dhms_duration import dhms_to_seconds
+from reconcile.utils.secret_reader import create_secret_reader
+
+if TYPE_CHECKING:
+    from mypy_boto3_ec2 import EC2Client
+else:
+    EC2Client = object
+
+QONTRACT_INTEGRATION = "aws_ami_cleanup"
+MANAGED_TAG = {"Key": "managed_by_integration", "Value": QONTRACT_INTEGRATION}
+
+
+class AWSAmi(BaseModel):
+    name: str
+    image_id: str
+    creation_date: datetime
+    snapshot_ids: list[str]
+
+    class Config:
+        frozen = True
+
+
+def get_aws_amis_from_launch_templates(ec2_client: EC2Client) -> set[str]:
+    amis = set()
+
+    paginator = ec2_client.get_paginator("describe_launch_templates")
+    for page in paginator.paginate():
+        for launch_template in page.get("LaunchTemplates", []):
+            if launch_template_id := launch_template.get("LaunchTemplateId"):
+                launch_template_versions = ec2_client.describe_launch_template_versions(
+                    LaunchTemplateId=launch_template_id,
+                    Versions=[str(launch_template.get("LatestVersionNumber"))],
+                ).get("LaunchTemplateVersions", [])
+
+                if launch_template_versions:
+                    if (
+                        ami_id := launch_template_versions[0]
+                        .get("LaunchTemplateData", {})
+                        .get("ImageId")
+                    ):
+                        amis.add(ami_id)
+
+    return amis
+
+
+def get_aws_amis(
+    ec2_client: EC2Client,
+    owner: str,
+    regex: str,
+    age_in_seconds: int,
+    utc_now: datetime,
+) -> list[AWSAmi]:
+    """Get amis that match regex older than given age"""
+
+    pattern = re.compile(regex)
+    paginator = ec2_client.get_paginator("describe_images")
+    results = []
+    for page in paginator.paginate(Owners=[owner]):
+        for image in page.get("Images", []):
+            if not re.search(pattern, image["Name"]):
+                continue
+
+            creation_date = datetime.strptime(
+                image["CreationDate"], "%Y-%m-%dT%H:%M:%S.%fZ"
+            )
+            current_delta = utc_now - creation_date
+            delete_delta = timedelta(seconds=age_in_seconds)
+
+            if current_delta < delete_delta:
+                continue
+
+            snapshot_ids = []
+            for bdv in image["BlockDeviceMappings"]:
+                ebs = bdv.get("Ebs")
+                if not ebs:
+                    continue
+
+                if sid := ebs.get("SnapshotId"):
+                    snapshot_ids.append(sid)
+
+            results.append(
+                AWSAmi(
+                    name=image["Name"],
+                    image_id=image["ImageId"],
+                    creation_date=creation_date,
+                    snapshot_ids=snapshot_ids,
+                )
+            )
+
+    return results
+
+
+def get_region(
+    config_region: str | None,
+    account_name: str,
+    resources_default_region: str,
+    supported_deployment_regions: list[str] | None,
+) -> str:
+    """Defines the region to search for AMIs."""
+    region = config_region or resources_default_region
+    if region not in (supported_deployment_regions or []):
+        raise ValueError(f"region {region} is not supported in {account_name}")
+
+    return region
+
+
+@defer
+def run(dry_run: bool, thread_pool_size: int, defer: Callable | None = None) -> None:
+    utc_now = datetime.utcnow()
+    gqlapi = gql.get_api()
+    aws_accounts = aws_accounts_query(gqlapi.query).accounts
+
+    # Get accounts that have cleanup configured
+    cleanup_accounts = []
+    for account in aws_accounts or []:
+        if not account.cleanup:
+            continue
+        is_ami_related = False
+        for cleanup in account.cleanup:
+            is_ami_related |= cleanup.provider == "ami"
+        if not is_ami_related:
+            continue
+        cleanup_accounts.append(account)
+
+    # Build a dict with all accounts that are used to share amis with. Together with
+    # the cleanup account we will look into the account's launch templates in order to
+    # find the AMIs that are currently being used. We will make sure that those are not
+    # deleted even if they have expired.
+    ami_accounts = defaultdict(set)
+    for account in cleanup_accounts:
+        ami_accounts[account.name].add(account.resources_default_region)
+
+        for sharing_config in account.sharing or []:
+            if not isinstance(sharing_config, AWSAccountSharingOptionAMIV1):
+                continue
+
+            region = get_region(
+                config_region=sharing_config.region,
+                account_name=sharing_config.account.name,
+                resources_default_region=sharing_config.account.resources_default_region,
+                supported_deployment_regions=sharing_config.account.supported_deployment_regions,
+            )
+
+            ami_accounts[sharing_config.account.name].add(
+                sharing_config.account.resources_default_region
+            )
+
+    # Build AWSApi object. We will use all those accounts listed in ami_accounts since
+    # we will also need to look for used AMIs.
+    accounts_dicted = [
+        account.dict(by_alias=True)
+        for account in aws_accounts or []
+        if account.name in ami_accounts
+    ]
+    secret_reader = create_secret_reader(
+        use_vault=get_app_interface_vault_settings().vault
+    )
+    aws_api = AWSApi(1, accounts_dicted, secret_reader=secret_reader, init_users=False)
+    if defer:  # defer is provided by the method decorator; this makes just mypy happy.
+        defer(aws_api.cleanup)
+
+    # Get all AMIs used
+    amis_used_in_launch_templates = set()
+    for account_name, regions in ami_accounts.items():
+        for region in regions:
+            session = aws_api.get_session(account_name)
+            ec2_client = aws_api.get_session_client(session, "ec2", region)
+            launch_template_amis = get_aws_amis_from_launch_templates(
+                ec2_client=ec2_client
+            )
+            if launch_template_amis:
+                amis_used_in_launch_templates.update(launch_template_amis)
+
+    # The action
+    for account in cleanup_accounts:
+        for cleanup_config in account.cleanup or []:
+            if not isinstance(cleanup_config, AWSAccountCleanupOptionAMIV1):
+                continue
+
+            region = get_region(
+                config_region=cleanup_config.region,
+                account_name=account.name,
+                resources_default_region=account.resources_default_region,
+                supported_deployment_regions=account.supported_deployment_regions,
+            )
+            age_in_seconds = dhms_to_seconds(cleanup_config.age)
+
+            session = aws_api.get_session(account.name)
+            ec2_client = aws_api.get_session_client(session, "ec2", region)
+
+            aws_amis = get_aws_amis(
+                ec2_client=ec2_client,
+                owner=account.uid,
+                regex=cleanup_config.regex,
+                age_in_seconds=age_in_seconds,
+                utc_now=utc_now,
+            )
+
+            for ami in aws_amis:
+                if ami.image_id in amis_used_in_launch_templates:
+                    logging.info(
+                        "Discarding AMI %s with id %s as it is still in use.",
+                        ami.name,
+                        ami.image_id,
+                    )
+                    continue
+
+                logging.info(
+                    "Deregistering image %s with id %s created in %s",
+                    ami.name,
+                    ami.image_id,
+                    ami.creation_date,
+                )
+
+                try:
+                    ec2_client.deregister_image(ImageId=ami.image_id, DryRun=dry_run)
+                except ClientError as e:
+                    if "DryRunOperation" in str(e):
+                        logging.info(e)
+                    else:
+                        raise
+
+                if not ami.snapshot_ids:
+                    continue
+
+                for snapshot_id in ami.snapshot_ids:
+                    logging.info(
+                        "Deleting associated snapshot %s from image %s",
+                        snapshot_id,
+                        ami.image_id,
+                    )
+
+                    try:
+                        ec2_client.delete_snapshot(
+                            SnapshotId=snapshot_id, DryRun=dry_run
+                        )
+                    except ClientError as e:
+                        if "DryRunOperation" in str(e):
+                            logging.info(e)
+                        else:
+                            raise

reconcile/aws_ami_share.py

@@ -62,14 +62,12 @@ def run(dry_run, defer=None):
                 src_ami_id = src_ami["image_id"]
                 found_dst_amis = [d for d in dst_amis if d["image_id"] == src_ami_id]
                 if not found_dst_amis:
-                    logging.info(
-                        [
-                            "share_ami",
-                            src_account["name"],
-                            dst_account["name"],
-                            src_ami_id,
-                        ]
-                    )
+                    logging.info([
+                        "share_ami",
+                        src_account["name"],
+                        dst_account["name"],
+                        src_ami_id,
+                    ])
                     if not dry_run:
                         aws_api.share_ami(
                             src_account, dst_account["uid"], src_ami_id, region
@@ -81,16 +79,22 @@ def run(dry_run, defer=None):
                 dst_ami_id = dst_ami["image_id"]
                 dst_ami_tags = dst_ami["tags"]
                 if MANAGED_TAG not in dst_ami_tags:
-                    logging.info(
-                        ["tag_shared_ami", dst_account["name"], dst_ami_id, MANAGED_TAG]
-                    )
+                    logging.info([
+                        "tag_shared_ami",
+                        dst_account["name"],
+                        dst_ami_id,
+                        MANAGED_TAG,
+                    ])
                     if not dry_run:
                         aws_api.create_tag(dst_account, dst_ami_id, MANAGED_TAG)
                 src_ami_tags = src_ami["tags"]
                 for src_tag in src_ami_tags:
                     if src_tag not in dst_ami_tags:
-                        logging.info(
-                            ["tag_shared_ami", dst_account["name"], dst_ami_id, src_tag]
-                        )
+                        logging.info([
+                            "tag_shared_ami",
+                            dst_account["name"],
+                            dst_ami_id,
+                            src_tag,
+                        ])
                         if not dry_run:
                             aws_api.create_tag(dst_account, dst_ami_id, src_tag)

reconcile/aws_cloudwatch_log_retention/integration.py

@@ -0,0 +1,253 @@
+import logging
+import re
+import typing
+from collections import defaultdict
+from collections.abc import Iterable
+from datetime import (
+    datetime,
+    timedelta,
+)
+from enum import Enum
+from typing import TYPE_CHECKING
+
+from botocore.exceptions import ClientError
+from pydantic import BaseModel
+
+from reconcile import queries
+from reconcile.queries import get_aws_accounts
+from reconcile.utils.aws_api import AWSApi
+
+if TYPE_CHECKING:
+    from mypy_boto3_logs import CloudWatchLogsClient  # type: ignore
+else:
+    CloudWatchLogsClient = object
+
+QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
+MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
+MANAGED_TAG = {MANAGED_BY_INTEGRATION_KEY: QONTRACT_INTEGRATION}
+DEFAULT_RETENTION_IN_DAYS = 90
+
+
+class AWSCloudwatchCleanupOption(BaseModel):
+    regex: typing.Pattern
+    retention_in_days: int
+    delete_empty_log_group: bool
+
+
+DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION = AWSCloudwatchCleanupOption(
+    regex=re.compile(".*"),
+    retention_in_days=DEFAULT_RETENTION_IN_DAYS,
+    delete_empty_log_group=False,
+)
+
+
+def get_desired_cleanup_options_by_region(
+    account: dict,
+) -> dict[str, list[AWSCloudwatchCleanupOption]]:
+    default_region = account["resourcesDefaultRegion"]
+    result = defaultdict(list)
+    if cleanup := account.get("cleanup"):
+        for cleanup_option in cleanup:
+            if cleanup_option["provider"] == "cloudwatch":
+                region = cleanup_option.get("region") or default_region
+                result[region].append(
+                    AWSCloudwatchCleanupOption(
+                        regex=re.compile(cleanup_option["regex"]),
+                        retention_in_days=cleanup_option["retention_in_days"],
+                        delete_empty_log_group=bool(
+                            cleanup_option["delete_empty_log_group"]
+                        ),
+                    )
+                )
+    if not result:
+        result[default_region].append(DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION)
+    return result
+
+
+def create_awsapi_client(accounts: list, thread_pool_size: int) -> AWSApi:
+    settings = queries.get_secret_reader_settings()
+    return AWSApi(thread_pool_size, accounts, settings=settings, init_users=False)
+
+
+def is_empty(
+    log_group: dict,
+) -> bool:
+    return log_group["storedBytes"] == 0
+
+
+def is_longer_than_retention(
+    log_group: dict,
+    desired_retention_days: int,
+) -> bool:
+    return (
+        datetime.fromtimestamp(log_group["creationTime"] / 1000)
+        + timedelta(days=desired_retention_days)
+        < datetime.utcnow()
+    )
+
+
+class TagStatus(Enum):
+    NOT_SET = "NOT_SET"
+    MANAGED_BY_CURRENT_INTEGRATION = "MANAGED_BY_CURRENT_INTEGRATION"
+    MANAGED_BY_OTHER_INTEGRATION = "MANAGED_BY_OTHER_INTEGRATION"
+
+
+def get_tag_status(
+    log_group: dict,
+    account_name: str,
+    region: str,
+    aws_api: AWSApi,
+) -> TagStatus:
+    tags = aws_api.get_cloudwatch_log_group_tags(
+        account_name,
+        log_group["arn"],
+        region,
+    )
+    managed_by_integration = tags.get(MANAGED_BY_INTEGRATION_KEY)
+    if managed_by_integration is None:
+        return TagStatus.NOT_SET
+    if managed_by_integration == QONTRACT_INTEGRATION:
+        return TagStatus.MANAGED_BY_CURRENT_INTEGRATION
+    return TagStatus.MANAGED_BY_OTHER_INTEGRATION
+
+
+def _reconcile_log_group(
+    dry_run: bool,
+    aws_log_group: dict,
+    desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
+    account_name: str,
+    region: str,
+    awsapi: AWSApi,
+) -> None:
+    current_retention_in_days = aws_log_group.get("retentionInDays")
+    log_group_name = aws_log_group["logGroupName"]
+    log_group_arn = aws_log_group["arn"]
+
+    desired_cleanup_option = _find_desired_cleanup_option(
+        log_group_name, desired_cleanup_options
+    )
+
+    if (
+        desired_cleanup_option.delete_empty_log_group
+        and is_empty(aws_log_group)
+        and is_longer_than_retention(
+            aws_log_group, desired_cleanup_option.retention_in_days
+        )
+    ):
+        if (
+            get_tag_status(aws_log_group, account_name, region, awsapi)
+            != TagStatus.MANAGED_BY_OTHER_INTEGRATION
+        ):
+            logging.info(
+                "Deleting empty log group %s",
+                log_group_arn,
+            )
+            if not dry_run:
+                awsapi.delete_cloudwatch_log_group(account_name, log_group_name, region)
+        return
+
+    if current_retention_in_days == desired_cleanup_option.retention_in_days:
+        return
+
+    match get_tag_status(aws_log_group, account_name, region, awsapi):
+        case TagStatus.MANAGED_BY_OTHER_INTEGRATION:
+            return
+        case TagStatus.MANAGED_BY_CURRENT_INTEGRATION:
+            pass
+        case TagStatus.NOT_SET:
+            logging.info(
+                "Setting tag %s for log group %s",
+                MANAGED_TAG,
+                log_group_arn,
+            )
+            if not dry_run:
+                awsapi.create_cloudwatch_tag(
+                    account_name, log_group_arn, MANAGED_TAG, region
+                )
+
+    logging.info(
+        "Setting %s retention days to %d",
+        log_group_arn,
+        desired_cleanup_option.retention_in_days,
+    )
+    if not dry_run:
+        awsapi.set_cloudwatch_log_retention(
+            account_name,
+            log_group_name,
+            desired_cleanup_option.retention_in_days,
+            region,
+        )
+
+
+def _find_desired_cleanup_option(
+    log_group_name: str,
+    desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
+) -> AWSCloudwatchCleanupOption:
+    """
+    Find the first cleanup option that regex matches the log group name.
+    If no match is found, return the default cleanup option.
+
+    :param log_group_name: The name of the log group
+    :param desired_cleanup_options: The desired cleanup options
+    :return: The desired cleanup option
+    """
+    return next(
+        (o for o in desired_cleanup_options if o.regex.match(log_group_name)),
+        DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION,
+    )
+
+
+def _reconcile_log_groups(
+    dry_run: bool,
+    aws_account: dict,
+    awsapi: AWSApi,
+) -> None:
+    account_name = aws_account["name"]
+    desired_cleanup_options_by_region = get_desired_cleanup_options_by_region(
+        aws_account
+    )
+    try:
+        for (
+            region,
+            desired_cleanup_options,
+        ) in desired_cleanup_options_by_region.items():
+            for aws_log_group in awsapi.get_cloudwatch_log_groups(
+                account_name,
+                region,
+            ):
+                _reconcile_log_group(
+                    dry_run=dry_run,
+                    aws_log_group=aws_log_group,
+                    desired_cleanup_options=desired_cleanup_options,
+                    account_name=account_name,
+                    region=region,
+                    awsapi=awsapi,
+                )
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "AccessDeniedException":
+            logging.info(
+                "Access denied for aws account %s. Skipping...",
+                account_name,
+            )
+        else:
+            logging.error(
+                "Error reconciling log groups for %s: %s",
+                account_name,
+                e,
+            )
+
+
+def get_active_aws_accounts() -> list[dict]:
+    return [
+        a
+        for a in get_aws_accounts(cleanup=True)
+        if "aws-cloudwatch-log-retention"
+        not in (a.get("disable") or {}).get("integrations", [])
+    ]
+
+
+def run(dry_run: bool, thread_pool_size: int) -> None:
+    aws_accounts = get_active_aws_accounts()
+    with create_awsapi_client(aws_accounts, thread_pool_size) as awsapi:
+        for aws_account in aws_accounts:
+            _reconcile_log_groups(dry_run, aws_account, awsapi)

reconcile/aws_iam_keys.py

@@ -26,7 +26,7 @@ def get_keys_to_delete(accounts) -> dict[str, list[str]]:
     return {
         account["name"]: account["deleteKeys"]
         for account in accounts
-        if account["deleteKeys"] not in (None, [])
+        if account["deleteKeys"]
     }
 
 

reconcile/aws_iam_password_reset.py

@@ -4,10 +4,9 @@ from collections.abc import (
     Iterable,
     Mapping,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
+
+from pydantic import BaseModel
 
 from reconcile import queries
 from reconcile.utils.aws_api import AWSApi
@@ -19,7 +18,7 @@ QONTRACT_INTEGRATION = "aws-iam-password-reset"
 
 def get_roles(
     roles: Iterable[Mapping[str, Any]], org_username: str
-) -> Optional[Mapping[str, Any]]:
+) -> Mapping[str, Any] | None:
     for d in roles:
         if d["org_username"] == org_username:
             return d
@@ -34,37 +33,74 @@ def account_in_roles(roles: Iterable[Mapping[str, Any]], aws_account: str) -> bo
     return False
 
 
+class AwsProfileToReset(BaseModel):
+    user_name: str
+    state_key: str
+
+
+class AwsAccountWithResets(BaseModel):
+    account: Mapping[str, Any]
+    resetPasswords: list[AwsProfileToReset]
+
+
 @defer
 def run(dry_run, defer=None):
     accounts = queries.get_aws_accounts(reset_passwords=True)
     settings = queries.get_app_interface_settings()
-    roles = queries.get_roles(aws=True)
+    roles = queries.get_roles(aws=True, saas_files=False)
     state = init_state(integration=QONTRACT_INTEGRATION)
     defer(state.cleanup)
 
+    accounts_to_reset: list[AwsAccountWithResets] = []
+
     for a in accounts:
         account_name = a["name"]
         reset_passwords = a.get("resetPasswords")
         if not reset_passwords:
             continue
-        with AWSApi(1, [a], settings=settings) as aws_api:
-            for r in reset_passwords:
-                user_name = r["user"]["org_username"]
-                request_id = r["requestId"]
-                state_key = f"{account_name}/{user_name}/{request_id}"
-                if state.exists(state_key):
-                    continue
 
-                role = get_roles(roles, user_name)
-                if not role:
-                    logging.error(f"Expected a role to be found with name {user_name}")
-                    sys.exit(1)
+        account_reset = AwsAccountWithResets(
+            account=a,
+            resetPasswords=[],
+        )
+        accounts_to_reset.append(account_reset)
 
-                if not account_in_roles(role["roles"], account_name):
-                    logging.error(f"User {user_name} is not in account {account_name}")
-                    sys.exit(1)
+        for r in reset_passwords:
+            user_name = r["user"]["org_username"]
+            request_id = r["requestId"]
+            state_key = f"{account_name}/{user_name}/{request_id}"
+            if state.exists(state_key):
+                continue
+
+            role = get_roles(roles, user_name)
+            if not role:
+                logging.error(f"Expected a role to be found with name {user_name}")
+                sys.exit(1)
+
+            if not account_in_roles(role["roles"], account_name):
+                logging.error(f"User {user_name} is not in account {account_name}")
+                sys.exit(1)
+
+            account_reset.resetPasswords.append(
+                AwsProfileToReset(
+                    user_name=user_name,
+                    state_key=state_key,
+                )
+            )
+
+    for a in accounts_to_reset:
+        if not a.resetPasswords:
+            continue
+
+        with AWSApi(1, [a.account], settings=settings) as aws_api:
+            for r in a.resetPasswords:
+                user_name = r.user_name
+                state_key = r.state_key
+
+                account_name = a.account["name"]
 
                 logging.info(["reset_password", account_name, user_name])
+
                 if dry_run:
                     continue