qontract-reconcile 0.10.2.dev349__py3-none-any.whl → 0.10.2.dev414__py3-none-any.whl

reconcile/aws_cloudwatch_log_retention/integration.py

@@ -2,31 +2,41 @@ from __future__ import annotations
 
 import logging
 import re
-import typing
 from collections import defaultdict
 from datetime import UTC, datetime, timedelta
-from enum import Enum
 from typing import TYPE_CHECKING
 
-from botocore.exceptions import ClientError
 from pydantic import BaseModel
 
-from reconcile import queries
 from reconcile.gql_definitions.aws_cloudwatch_log_retention.aws_accounts import (
     AWSAccountCleanupOptionCloudWatchV1,
     AWSAccountV1,
 )
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
 from reconcile.typed_queries.aws_cloudwatch_log_retention.aws_accounts import (
     get_aws_accounts,
 )
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import gql
-from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
+from reconcile.utils.datetime_util import utc_now
+from reconcile.utils.differ import diff_mappings
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.state import init_state
+
+TAGS_KEY = "tags.json"
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
     from mypy_boto3_logs.type_defs import LogGroupTypeDef
 
+    from reconcile.utils.aws_api_typed.logs import AWSApiLogs
+    from reconcile.utils.gql import GqlApi
+
 
 QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
 MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
@@ -35,7 +45,7 @@ DEFAULT_RETENTION_IN_DAYS = 90
 
 
 class AWSCloudwatchCleanupOption(BaseModel):
-    regex: typing.Pattern
+    regex: re.Pattern
     retention_in_days: int
     delete_empty_log_group: bool
 
@@ -67,16 +77,6 @@ def get_desired_cleanup_options_by_region(
     return result
 
 
-def create_awsapi_client(accounts: list[AWSAccountV1], thread_pool_size: int) -> AWSApi:
-    settings = queries.get_secret_reader_settings()
-    return AWSApi(
-        thread_pool_size,
-        [account.dict(by_alias=True) for account in accounts],
-        settings=settings,
-        init_users=False,
-    )
-
-
 def is_empty(log_group: LogGroupTypeDef) -> bool:
     return log_group["storedBytes"] == 0
 
@@ -85,47 +85,32 @@ def is_longer_than_retention(
     log_group: LogGroupTypeDef,
     desired_retention_days: int,
 ) -> bool:
-    return datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC) + timedelta(
-        days=desired_retention_days
-    ) < datetime.now(tz=UTC)
-
-
-class TagStatus(Enum):
-    NOT_SET = "NOT_SET"
-    MANAGED_BY_CURRENT_INTEGRATION = "MANAGED_BY_CURRENT_INTEGRATION"
-    MANAGED_BY_OTHER_INTEGRATION = "MANAGED_BY_OTHER_INTEGRATION"
+    return (
+        datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC)
+        + timedelta(days=desired_retention_days)
+        < utc_now()
+    )
 
 
-def get_tag_status(
-    log_group: LogGroupTypeDef,
-    account_name: str,
-    region: str,
-    aws_api: AWSApi,
-) -> TagStatus:
-    tags = aws_api.get_cloudwatch_log_group_tags(
-        account_name,
-        log_group["arn"],
-        region,
-    )
+def _is_managed_by_other_integration(tags: dict[str, str]) -> bool:
     managed_by_integration = tags.get(MANAGED_BY_INTEGRATION_KEY)
-    if managed_by_integration is None:
-        return TagStatus.NOT_SET
-    if managed_by_integration == QONTRACT_INTEGRATION:
-        return TagStatus.MANAGED_BY_CURRENT_INTEGRATION
-    return TagStatus.MANAGED_BY_OTHER_INTEGRATION
+    return (
+        managed_by_integration is not None
+        and managed_by_integration != QONTRACT_INTEGRATION
+    )
 
 
 def _reconcile_log_group(
     dry_run: bool,
-    aws_log_group: LogGroupTypeDef,
+    log_group: LogGroupTypeDef,
     desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
-    account_name: str,
-    region: str,
-    awsapi: AWSApi,
+    desired_tags: dict[str, str],
+    last_tags: dict[str, str],
+    aws_api_logs: AWSApiLogs,
 ) -> None:
-    current_retention_in_days = aws_log_group.get("retentionInDays")
-    log_group_name = aws_log_group["logGroupName"]
-    log_group_arn = aws_log_group["arn"]
+    current_retention_in_days = log_group.get("retentionInDays")
+    log_group_name = log_group["logGroupName"]
+    log_group_arn = log_group["arn"]
 
     desired_cleanup_option = _find_desired_cleanup_option(
         log_group_name, desired_cleanup_options
@@ -133,54 +118,66 @@ def _reconcile_log_group(
 
     if (
         desired_cleanup_option.delete_empty_log_group
-        and is_empty(aws_log_group)
+        and is_empty(log_group)
         and is_longer_than_retention(
-            aws_log_group, desired_cleanup_option.retention_in_days
+            log_group, desired_cleanup_option.retention_in_days
         )
     ):
-        if (
-            get_tag_status(aws_log_group, account_name, region, awsapi)
-            != TagStatus.MANAGED_BY_OTHER_INTEGRATION
-        ):
+        tags = aws_api_logs.get_tags(log_group_arn)
+        if not _is_managed_by_other_integration(tags):
             logging.info(
                 "Deleting empty log group %s",
                 log_group_arn,
             )
             if not dry_run:
-                awsapi.delete_cloudwatch_log_group(account_name, log_group_name, region)
+                aws_api_logs.delete_log_group(log_group_name)
         return
 
-    if current_retention_in_days == desired_cleanup_option.retention_in_days:
+    if (
+        current_retention_in_days == desired_cleanup_option.retention_in_days
+        and last_tags == desired_tags
+    ):
         return
 
-    match get_tag_status(aws_log_group, account_name, region, awsapi):
-        case TagStatus.MANAGED_BY_OTHER_INTEGRATION:
-            return
-        case TagStatus.MANAGED_BY_CURRENT_INTEGRATION:
-            pass
-        case TagStatus.NOT_SET:
-            logging.info(
-                "Setting tag %s for log group %s",
-                MANAGED_TAG,
+    current_tags = aws_api_logs.get_tags(log_group_arn)
+    if _is_managed_by_other_integration(current_tags):
+        return
+
+    diff_result = diff_mappings(
+        current=current_tags,
+        desired=desired_tags,
+    )
+    if to_delete := diff_result.delete.keys() & last_tags.keys():
+        logging.info(
+            "Deleting tags %s for log group %s",
+            to_delete,
+            log_group_arn,
+        )
+        if not dry_run:
+            aws_api_logs.delete_tags(
                 log_group_arn,
+                to_delete,
             )
-            if not dry_run:
-                awsapi.create_cloudwatch_tag(
-                    account_name, log_group_arn, MANAGED_TAG, region
-                )
+    if diff_result.add or diff_result.change:
+        logging.info(
+            "Setting tags %s for log group %s",
+            desired_tags,
+            log_group_arn,
+        )
+        if not dry_run:
+            aws_api_logs.set_tags(log_group_arn, desired_tags)
 
-    logging.info(
-        "Setting %s retention days to %d",
-        log_group_arn,
-        desired_cleanup_option.retention_in_days,
-    )
-    if not dry_run:
-        awsapi.set_cloudwatch_log_retention(
-            account_name,
-            log_group_name,
+    if current_retention_in_days != desired_cleanup_option.retention_in_days:
+        logging.info(
+            "Setting %s retention days to %d",
+            log_group_arn,
             desired_cleanup_option.retention_in_days,
-            region,
         )
+        if not dry_run:
+            aws_api_logs.put_retention_policy(
+                log_group_name,
+                desired_cleanup_option.retention_in_days,
+            )
 
 
 def _find_desired_cleanup_option(
@@ -191,60 +188,63 @@ def _find_desired_cleanup_option(
     Find the first cleanup option that regex matches the log group name.
     If no match is found, return the default cleanup option.
 
-    :param log_group_name: The name of the log group
-    :param desired_cleanup_options: The desired cleanup options
-    :return: The desired cleanup option
+    Args:
+        log_group_name: The name of the log group.
+        desired_cleanup_options: A list of desired cleanup options.
+    Returns:
+        The matching cleanup option or the default one.
     """
-    return next(
-        (o for o in desired_cleanup_options if o.regex.match(log_group_name)),
-        DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION,
-    )
+    for option in desired_cleanup_options:
+        if option.regex.match(log_group_name):
+            return option
+    return DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION
 
 
 def _reconcile_log_groups(
     dry_run: bool,
     aws_account: AWSAccountV1,
-    awsapi: AWSApi,
-) -> None:
-    account_name = aws_account.name
-    desired_cleanup_options_by_region = get_desired_cleanup_options_by_region(
-        aws_account
+    last_tags: dict[str, str],
+    default_tags: dict[str, str],
+    automation_token: dict[str, str],
+) -> dict[str, str]:
+    desired_tags = (
+        default_tags | get_aws_account_tags(aws_account.organization) | MANAGED_TAG
     )
-    try:
-        for (
-            region,
-            desired_cleanup_options,
-        ) in desired_cleanup_options_by_region.items():
-            for aws_log_group in awsapi.get_cloudwatch_log_groups(
-                account_name,
-                region,
-            ):
-                _reconcile_log_group(
-                    dry_run=dry_run,
-                    aws_log_group=aws_log_group,
-                    desired_cleanup_options=desired_cleanup_options,
-                    account_name=account_name,
-                    region=region,
-                    awsapi=awsapi,
+    for (
+        region,
+        desired_cleanup_options,
+    ) in get_desired_cleanup_options_by_region(aws_account).items():
+        aws_credentials = AWSStaticCredentials(
+            access_key_id=automation_token["aws_access_key_id"],
+            secret_access_key=automation_token["aws_secret_access_key"],
+            region=region,
+        )
+        with AWSApi(aws_credentials) as aws_api:
+            aws_api_logs = aws_api.logs
+            try:
+                for log_group in aws_api_logs.get_log_groups():
+                    _reconcile_log_group(
+                        dry_run=dry_run,
+                        log_group=log_group,
+                        desired_cleanup_options=desired_cleanup_options,
+                        desired_tags=desired_tags,
+                        last_tags=last_tags,
+                        aws_api_logs=aws_api_logs,
+                    )
+            except aws_api_logs.client.exceptions.ClientError as e:
+                logging.error(
+                    "Error reconciling log groups for %s: %s",
+                    aws_account.name,
+                    e,
                 )
-    except ClientError as e:
-        if e.response["Error"]["Code"] == "AccessDeniedException":
-            logging.info(
-                "Access denied for aws account %s. Skipping...",
-                account_name,
-            )
-        else:
-            logging.error(
-                "Error reconciling log groups for %s: %s",
-                account_name,
-                e,
-            )
+                return last_tags
+    return desired_tags
 
 
-def get_active_aws_accounts() -> list[AWSAccountV1]:
+def get_active_aws_accounts(gql_api: GqlApi) -> list[AWSAccountV1]:
     return [
         account
-        for account in get_aws_accounts(gql.get_api())
+        for account in get_aws_accounts(gql_api)
         if not (
             account.disable
             and account.disable.integrations
@@ -253,8 +253,37 @@ def get_active_aws_accounts() -> list[AWSAccountV1]:
     ]
 
 
-def run(dry_run: bool, thread_pool_size: int) -> None:
-    aws_accounts = get_active_aws_accounts()
-    with create_awsapi_client(aws_accounts, thread_pool_size) as awsapi:
-        for aws_account in aws_accounts:
-            _reconcile_log_groups(dry_run, aws_account, awsapi)
+def get_default_tags(gql_api: GqlApi) -> dict[str, str]:
+    try:
+        return get_settings(gql_api.query).default_tags
+    except ValueError:
+        # no settings found
+        return {}
+
+
+def run(dry_run: bool) -> None:
+    gql_api = gql.get_api()
+    aws_accounts = get_active_aws_accounts(gql_api)
+    vault_settings = get_app_interface_vault_settings(query_func=gql_api.query)
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    default_tags = get_default_tags(gql_api)
+
+    with init_state(
+        integration=QONTRACT_INTEGRATION,
+        secret_reader=secret_reader,
+    ) as state:
+        last_tags = state.get(TAGS_KEY, {})
+        desired_tags = {
+            aws_account.name: _reconcile_log_groups(
+                dry_run=dry_run,
+                aws_account=aws_account,
+                last_tags=last_tags.get(aws_account.name, {}),
+                default_tags=default_tags,
+                automation_token=secret_reader.read_all_secret(
+                    aws_account.automation_token
+                ),
+            )
+            for aws_account in aws_accounts
+        }
+        if not dry_run and desired_tags != last_tags:
+            state.add(TAGS_KEY, desired_tags, force=True)

reconcile/aws_ecr_image_pull_secrets.py

@@ -1,11 +1,11 @@
 import base64
-import json
 import logging
 from collections.abc import Mapping
 from typing import Any
 
 from reconcile import queries
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.vault import VaultClient
 
 QONTRACT_INTEGRATION = "aws-ecr-image-pull-secrets"
@@ -35,7 +35,7 @@ def construct_dockercfg_secret_data(data: Mapping[str, Any]) -> dict[str, str]:
         }
     }
 
-    return {".dockerconfigjson": enc_dec(json.dumps(data))}
+    return {".dockerconfigjson": enc_dec(json_dumps(data))}
 
 
 def construct_basic_auth_secret_data(data: Mapping[str, Any]) -> dict[str, str]:

reconcile/aws_iam_keys.py

@@ -17,7 +17,7 @@ QONTRACT_INTEGRATION = "aws-iam-keys"
 def filter_accounts(
     accounts: Iterable[dict[str, Any]], account_name: str | None
 ) -> list[dict[str, Any]]:
-    accounts = [a for a in accounts if a.get("deleteKeys") or a.get("disableKeys")]
+    accounts = [a for a in accounts if a.get("deleteKeys")]
     if account_name:
         accounts = [a for a in accounts if a["name"] == account_name]
     return accounts
@@ -31,43 +31,17 @@ def get_keys_to_delete(accounts: Iterable[dict[str, Any]]) -> dict[str, list[str
     }
 
 
-def get_keys_to_disable(accounts: Iterable[dict[str, Any]]) -> dict[str, list[str]]:
-    return {
-        account["name"]: account["disableKeys"]
-        for account in accounts
-        if account.get("disableKeys")
-    }
-
-
-def should_run(
-    state: State,
-    keys_to_delete: dict[str, list[str]],
-    keys_to_disable: dict[str, list[str]] | None = None,
-) -> bool:
+def should_run(state: State, keys_to_delete: dict[str, list[str]]) -> bool:
     for account_name, keys in keys_to_delete.items():
         if state.get(account_name, []) != keys:
             return True
-    if keys_to_disable:
-        for account_name, keys in keys_to_disable.items():
-            disable_state_key = f"{account_name}_disable"
-            if state.get(disable_state_key, []) != keys:
-                return True
     return False
 
 
-def update_state(
-    state: State,
-    keys_to_update: dict[str, list[str]],
-    keys_to_disable: dict[str, list[str]] | None = None,
-) -> None:
+def update_state(state: State, keys_to_update: dict[str, list[str]]) -> None:
     for account_name, keys in keys_to_update.items():
         if state.get(account_name, []) != keys:
             state.add(account_name, keys, force=True)
-    if keys_to_disable:
-        for account_name, keys in keys_to_disable.items():
-            disable_state_key = f"{account_name}_disable"
-            if state.get(disable_state_key, []) != keys:
-                state.add(disable_state_key, keys, force=True)
 
 
 def init_tf_working_dirs(
@@ -91,6 +65,7 @@ def init_tf_working_dirs(
         thread_pool_size,
         accounts,
         settings=settings,
+        default_tags=None,
     )
     return ts.dump()
 
@@ -122,8 +97,7 @@ def run(
     if defer:
         defer(state.cleanup)
     keys_to_delete = get_keys_to_delete(accounts)
-    keys_to_disable = get_keys_to_disable(accounts)
-    if not should_run(state, keys_to_delete, keys_to_disable):
+    if not should_run(state, keys_to_delete):
         logging.debug("nothing to do here")
         # using return because terraform-resources
         # may be the calling entity, and has more to do
@@ -137,10 +111,6 @@ def run(
         error, service_account_recycle_complete = aws.delete_keys(
             dry_run, keys_to_delete, working_dirs, disable_service_account_keys
         )
-        if keys_to_disable:
-            disable_error = aws.disable_keys(dry_run, keys_to_disable)
-            # combine errors from both operations
-            error = error or disable_error
     if error:
         sys.exit(1)
 
@@ -149,12 +119,8 @@ def run(
         and not disable_service_account_keys
         and service_account_recycle_complete
     ):
-        update_state(state, keys_to_delete, keys_to_disable)
+        update_state(state, keys_to_delete)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
-    accounts = queries.get_aws_accounts(terraform_state=True)
-    return {
-        "keys": get_keys_to_delete(accounts),
-        "disable_keys": get_keys_to_disable(accounts),
-    }
+    return {"keys": get_keys_to_delete(queries.get_aws_accounts(terraform_state=True))}

reconcile/aws_saml_idp/integration.py

@@ -19,6 +19,7 @@ from reconcile.gql_definitions.aws_saml_idp.aws_accounts import (
     query as aws_accounts_query,
 )
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
@@ -81,7 +82,7 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
         if not query_func:
             query_func = gql.get_api().query
         return {
-            "accounts": [c.dict() for c in self.get_aws_accounts(query_func)],
+            "accounts": [c.model_dump() for c in self.get_aws_accounts(query_func)],
         }
 
     def get_aws_accounts(
@@ -124,20 +125,27 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
         aws_accounts = self.get_aws_accounts(
             gql_api.query, account_name=self.params.account_name
         )
-        aws_accounts_dict = [account.dict(by_alias=True) for account in aws_accounts]
-
+        aws_accounts_dict = [
+            account.model_dump(by_alias=True) for account in aws_accounts
+        ]
+        try:
+            default_tags = get_settings().default_tags
+        except ValueError:
+            # no external resources settings found
+            default_tags = None
         ts = TerrascriptClient(
             self.name.replace("-", "_"),
             "",
             self.params.thread_pool_size,
             aws_accounts_dict,
             secret_reader=self.secret_reader,
+            default_tags=default_tags,
         )
 
         for saml_idp_config in self.build_saml_idp_config(
             aws_accounts,
             saml_idp_name=self.params.saml_idp_name,
-            saml_metadata=self.get_saml_metadata(self.params.saml_metadata_url),
+            saml_metadata=self.get_saml_metadata(str(self.params.saml_metadata_url)),
         ):
             ts.populate_saml_idp(
                 account_name=saml_idp_config.account_name,