qontract-reconcile 0.10.2.dev345__py3-none-any.whl → 0.10.2.dev408__py3-none-any.whl

reconcile/terraform_aws_route53.py

@@ -9,6 +9,7 @@ from typing import Any
 
 from reconcile import queries
 from reconcile.status import ExitCodes
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import dnsutils
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.constants import DEFAULT_THREAD_POOL_SIZE
@@ -227,13 +228,18 @@ def run(
             f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
         )
         return
-
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         "",
         thread_pool_size,
         participating_accounts,
         settings=settings,
+        default_tags=default_tags,
     )
 
     desired_state = build_desired_state(zones, all_accounts, settings)

reconcile/terraform_init/integration.py

@@ -1,6 +1,5 @@
 import logging
 from collections.abc import Callable
-from datetime import UTC, datetime
 from typing import Any
 
 import jinja2
@@ -12,12 +11,16 @@ from reconcile.gql_definitions.terraform_init.aws_accounts import (
 from reconcile.terraform_init.merge_request import Renderer, create_parser
 from reconcile.terraform_init.merge_request_manager import MergeRequestManager, MrData
 from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
 from reconcile.utils.aws_api_typed.api import AWSApi, AWSStaticCredentials
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.gql import GqlApi
 from reconcile.utils.runtime.integration import (
     PydanticRunParams,
     QontractReconcileIntegration,
@@ -35,6 +38,12 @@ class TerraformInitIntegrationParams(PydanticRunParams):
     # To avoid the accidental deletion of the resource file, explicitly set the
     # qontract.cli option in the integration extraArgs!
     state_tmpl_resource: str = "/terraform-init/terraform-state.yml"
+    cloudformation_template_resource: str = (
+        "/terraform-init/terraform-state-s3-bucket.yaml"
+    )
+    cloudformation_import_template_resource: str = (
+        "/terraform-init/terraform-state-s3-bucket-import.yaml"
+    )
     template_collection_root_path: str = "data/templating/collections/terraform-init"
 
 
@@ -62,18 +71,28 @@ class TerraformInitIntegration(
     def get_aws_accounts(
         self, query_func: Callable, account_name: str | None = None
     ) -> list[AWSAccountV1]:
-        """Return all AWS accounts with terraform username but no terraform state set."""
+        """Return all AWS accounts with terraform username."""
         return [
             account
             for account in aws_accounts_query(query_func).accounts or []
             if integration_is_enabled(self.name, account)
             and (not account_name or account.name == account_name)
             and account.terraform_username
-            and not account.terraform_state
         ]
 
+    @staticmethod
+    def get_default_tags(gql_api: GqlApi) -> dict[str, str]:
+        try:
+            return get_settings(gql_api.query).default_tags
+        except ValueError:
+            # no settings found
+            return {}
+
+    @staticmethod
     def render_state_collection(
-        self, template: str, bucket_name: str, account: AWSAccountV1
+        template: str,
+        bucket_name: str,
+        account: AWSAccountV1,
     ) -> str:
         return jinja2.Template(
             template,
@@ -85,24 +104,114 @@ class TerraformInitIntegration(
             "account_name": account.name,
             "bucket_name": bucket_name,
             "region": account.resources_default_region,
-            "timestamp": int(datetime.now(tz=UTC).timestamp()),
+            "timestamp": int(utc_now().timestamp()),
         })
 
     def reconcile_account(
         self,
-        account_aws_api: AWSApi,
+        aws_api: AWSApi,
+        merge_request_manager: MergeRequestManager,
+        dry_run: bool,
+        account: AWSAccountV1,
+        state_template: str,
+        cloudformation_template: str,
+        cloudformation_import_template: str,
+        default_tags: dict[str, str],
+    ) -> None:
+        """
+        Reconcile the terraform state for a given account.
+
+        Create S3 bucket via CloudFormation and Merge Request to update template file on init.
+        Import existing bucket if it exists but not managed by CloudFormation.
+        Update CloudFormation stack if tags or template body differ.
+
+        CloudFormation stack name and bucket name is `terraform-<account_name>`.
+        `cloudformation_import_template` should be minimal template with identifier fields only.
+        `cloudformation_template` should be the full template.
+        Use import template to import then update stack to match full template.
+        This will ensure imported resources match CloudFormation template.
+        And all desired changes in full template are applied.
+        Both templates must have BucketName in Parameters.
+
+        Args:
+            aws_api: AWSApi: AWS API client for the target account.
+            merge_request_manager: MergeRequestManager: Manager to handle merge requests.
+            dry_run: bool: If True, do not make any changes.
+            account: AWSAccountV1: The AWS account to reconcile.
+            state_template: str: Jinja2 template for the Terraform state configuration.
+            cloudformation_template: str: CloudFormation template to create the S3 bucket.
+            cloudformation_import_template: str: CloudFormation template to import existing S3 bucket.
+            default_tags: dict[str, str]: Default tags to apply to the CloudFormation stack.
+
+        Returns:
+            None
+        """
+        bucket_name = (
+            account.terraform_state.bucket
+            if account.terraform_state
+            else f"terraform-{account.name}"
+        )
+
+        tags = default_tags | get_aws_account_tags(account.organization)
+
+        if account.terraform_state is None:
+            return self._provision_terraform_state(
+                aws_api=aws_api,
+                merge_request_manager=merge_request_manager,
+                dry_run=dry_run,
+                account=account,
+                bucket_name=bucket_name,
+                cloudformation_template=cloudformation_template,
+                state_template=state_template,
+                tags=tags,
+            )
+
+        stack = aws_api.cloudformation.get_stack(stack_name=bucket_name)
+
+        if stack is None:
+            return self._import_cloudformation_stack(
+                aws_api=aws_api,
+                dry_run=dry_run,
+                bucket_name=bucket_name,
+                cloudformation_import_template=cloudformation_import_template,
+                cloudformation_template=cloudformation_template,
+                tags=tags,
+            )
+
+        return self._reconcile_cloudformation_stack(
+            aws_api=aws_api,
+            dry_run=dry_run,
+            bucket_name=bucket_name,
+            cloudformation_template=cloudformation_template,
+            tags=tags,
+            current_tags={tag["Key"]: tag["Value"] for tag in stack.get("Tags", [])},
+        )
+
+    def _provision_terraform_state(
+        self,
+        aws_api: AWSApi,
         merge_request_manager: MergeRequestManager,
         dry_run: bool,
-        state_collection: str,
-        bucket_name: str,
         account: AWSAccountV1,
+        bucket_name: str,
+        cloudformation_template: str,
+        state_template: str,
+        tags: dict[str, str],
     ) -> None:
         logging.info("Creating bucket '%s' for account '%s'", bucket_name, account.name)
         if not dry_run:
-            # the creation of the bucket is idempotent
-            account_aws_api.s3.create_bucket(
-                name=bucket_name, region=account.resources_default_region
+            aws_api.cloudformation.create_stack(
+                stack_name=bucket_name,
+                change_set_name=f"create-{bucket_name}",
+                template_body=cloudformation_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
             )
+        state_collection = self.render_state_collection(
+            template=state_template,
+            bucket_name=bucket_name,
+            account=account,
+        )
         merge_request_manager.create_merge_request(
             data=MrData(
                 account=account.name,
@@ -111,6 +220,55 @@ class TerraformInitIntegration(
             )
         )
 
+    @staticmethod
+    def _import_cloudformation_stack(
+        aws_api: AWSApi,
+        dry_run: bool,
+        bucket_name: str,
+        cloudformation_import_template: str,
+        cloudformation_template: str,
+        tags: dict[str, str],
+    ) -> None:
+        logging.info("Importing existing bucket %s", bucket_name)
+        if not dry_run:
+            aws_api.cloudformation.create_stack(
+                stack_name=bucket_name,
+                change_set_name=f"import-{bucket_name}",
+                template_body=cloudformation_import_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
+            )
+            logging.info("Syncing stack %s after import", bucket_name)
+            aws_api.cloudformation.update_stack(
+                stack_name=bucket_name,
+                template_body=cloudformation_template,
+                parameters={"BucketName": bucket_name},
+                tags=tags,
+            )
+
+    @staticmethod
+    def _reconcile_cloudformation_stack(
+        aws_api: AWSApi,
+        dry_run: bool,
+        bucket_name: str,
+        cloudformation_template: str,
+        tags: dict[str, str],
+        current_tags: dict[str, str],
+    ) -> None:
+        if (
+            current_tags != tags
+            or aws_api.cloudformation.get_template_body(stack_name=bucket_name)
+            != cloudformation_template
+        ):
+            logging.info("Updating stack %s", bucket_name)
+            if not dry_run:
+                aws_api.cloudformation.update_stack(
+                    stack_name=bucket_name,
+                    template_body=cloudformation_template,
+                    parameters={"BucketName": bucket_name},
+                    tags=tags,
+                )
+
     @defer
     def run(self, dry_run: bool, defer: Callable | None = None) -> None:
         """Run the integration."""
@@ -144,6 +302,14 @@ class TerraformInitIntegration(
         state_template = gql_api.get_resource(path=self.params.state_tmpl_resource)[
             "content"
         ]
+        cloudformation_template = gql_api.get_resource(
+            path=self.params.cloudformation_template_resource
+        )["content"]
+        cloudformation_import_template = gql_api.get_resource(
+            path=self.params.cloudformation_import_template_resource
+        )["content"]
+        default_tags = self.get_default_tags(gql_api)
+
         for account in accounts:
             secret = self.secret_reader.read_all_secret(account.automation_token)
             with AWSApi(
@@ -153,15 +319,13 @@ class TerraformInitIntegration(
                     region=account.resources_default_region,
                 )
             ) as account_aws_api:
-                bucket_name = f"terraform-{account.name}"
-                state_collection = self.render_state_collection(
-                    state_template, bucket_name, account
-                )
                 self.reconcile_account(
-                    account_aws_api,
-                    merge_request_manager,
-                    dry_run,
-                    state_collection,
-                    bucket_name,
-                    account,
+                    aws_api=account_aws_api,
+                    merge_request_manager=merge_request_manager,
+                    dry_run=dry_run,
+                    account=account,
+                    state_template=state_template,
+                    cloudformation_template=cloudformation_template,
+                    cloudformation_import_template=cloudformation_import_template,
+                    default_tags=default_tags,
                 )

reconcile/terraform_resources.py

@@ -27,6 +27,7 @@ from reconcile.gql_definitions.terraform_resources.terraform_resources_namespace
 from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.terraform_namespaces import get_namespaces
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
@@ -158,6 +159,7 @@ def init_working_dirs(
     accounts: list[dict[str, Any]],
     thread_pool_size: int,
     settings: Mapping[str, Any] | None = None,
+    default_tags: Mapping[str, str] | None = None,
 ) -> tuple[Terrascript, dict[str, str]]:
     ts = Terrascript(
         QONTRACT_INTEGRATION,
@@ -165,6 +167,7 @@ def init_working_dirs(
         thread_pool_size,
         accounts,
         settings=settings,
+        default_tags=default_tags,
     )
     working_dirs = ts.dump()
     return ts, working_dirs
@@ -241,8 +244,15 @@ def setup(
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
     settings = queries.get_app_interface_settings() or {}
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     # initialize terrascript (scripting engine to generate terraform manifests)
-    ts, working_dirs = init_working_dirs(accounts, thread_pool_size, settings=settings)
+    ts, working_dirs = init_working_dirs(
+        accounts, thread_pool_size, settings=settings, default_tags=default_tags
+    )
 
     # initialize terraform client
     # it is used to plan and apply according to the output of terrascript

reconcile/terraform_tgw_attachments.py

@@ -32,6 +32,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.clusters_with_peering import get_clusters_with_peering
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.terraform_tgw_attachments.aws_accounts import (
     get_aws_accounts,
 )
@@ -444,13 +445,18 @@ def setup(
         raise RuntimeError("Could not find VPC ID for cluster")
 
     _validate_tgw_connection_names(desired_state)
-
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         "",
         thread_pool_size,
         tgw_accounts,
         settings=vault_settings.dict(by_alias=True),
+        default_tags=default_tags,
     )
     tgw_rosa_cluster_accounts = [
         account_by_name[c.spec.account.name]

reconcile/terraform_users.py

@@ -11,6 +11,7 @@ from reconcile import (
 )
 from reconcile.change_owners.diff import IDENTIFIER_FIELD_NAME
 from reconcile.gql_definitions.common.pgp_reencryption_settings import query
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.utils import (
     expiration,
     gql,
@@ -126,12 +127,18 @@ def setup(
     participating_aws_accounts = _filter_participating_aws_accounts(accounts, roles)
 
     settings = queries.get_app_interface_settings()
+    try:
+        default_tags = get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     ts = Terrascript(
         QONTRACT_INTEGRATION,
         QONTRACT_TF_PREFIX,
         thread_pool_size,
         participating_aws_accounts,
         settings=settings,
+        default_tags=default_tags,
     )
     err = ts.populate_users(
         roles,

reconcile/terraform_vpc_peerings.py

@@ -7,6 +7,7 @@ from typing import Any, TypedDict
 import reconcile.utils.terraform_client as terraform
 import reconcile.utils.terrascript_aws_client as terrascript
 from reconcile import queries
+from reconcile.typed_queries import external_resources
 from reconcile.utils import (
     aws_api,
     ocm,
@@ -654,8 +655,18 @@ def run(
         ])
 
     account_by_name = {a["name"]: a for a in accounts}
+    try:
+        default_tags = external_resources.get_settings().default_tags
+    except ValueError:
+        # no external resources settings found
+        default_tags = None
     with terrascript.TerrascriptClient(
-        QONTRACT_INTEGRATION, "", thread_pool_size, infra_accounts, settings=settings
+        QONTRACT_INTEGRATION,
+        "",
+        thread_pool_size,
+        infra_accounts,
+        settings=settings,
+        default_tags=default_tags,
     ) as ts:
         rosa_cluster_accounts = [
             account_by_name[c["spec"]["account"]["name"]]
@@ -664,8 +675,8 @@ def run(
         ]
         ts.populate_configs(rosa_cluster_accounts)
 
-        for infra_account_name, items in participating_accounts.items():
-            ts.populate_additional_providers(infra_account_name, items)
+        for infra_account_name, accounts in participating_accounts.items():
+            ts.populate_additional_providers(infra_account_name, accounts)
         ts.populate_vpc_peerings(desired_state)
         working_dirs = ts.dump(print_to_file=print_to_file)
         terraform_configurations = ts.terraform_configurations()

reconcile/terraform_vpc_resources/integration.py

@@ -20,6 +20,7 @@ from reconcile.typed_queries.app_interface_vault_settings import (
     get_app_interface_vault_settings,
 )
 from reconcile.typed_queries.aws_vpc_requests import get_aws_vpc_requests
+from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
 from reconcile.utils import gql
@@ -162,12 +163,18 @@ class TerraformVpcResources(QontractReconcileIntegration[TerraformVpcResourcesPa
             sys.exit(ExitCodes.SUCCESS)
 
         accounts_untyped: list[dict] = [acc.dict(by_alias=True) for acc in accounts]
+        try:
+            default_tags = get_settings().default_tags
+        except ValueError:
+            # no external resources settings found
+            default_tags = None
         with TerrascriptClient(
             integration=QONTRACT_INTEGRATION,
             integration_prefix=QONTRACT_TF_PREFIX,
             thread_pool_size=thread_pool_size,
             accounts=accounts_untyped,
             secret_reader=secret_reader,
+            default_tags=default_tags,
         ) as ts_client:
             ts_client.populate_vpc_requests(data, AWS_PROVIDER_VERSION)
 

reconcile/typed_queries/aws_account_tags.py

@@ -0,0 +1,41 @@
+import json
+from collections.abc import Mapping
+
+from reconcile.gql_definitions.fragments.aws_organization import (
+    AWSOrganization,
+)
+
+
+def get_aws_account_tags(
+    organization: AWSOrganization | Mapping | None,
+) -> dict[str, str]:
+    """
+    Get AWS account tags by merging payer account tags
+
+    Args:
+        organization: AWSOrganization | Mapping | None - The organization object from which to extract tags.
+
+    Returns:
+        dict[str, str]: A dictionary containing the merged tags from the payer account and the account itself.
+    """
+    if organization is None:
+        return {}
+
+    match organization:
+        case AWSOrganization():
+            payer_account_tags = (
+                organization.payer_account.organization_account_tags or {}
+            )
+            account_tags = organization.tags or {}
+        case Mapping():
+            payer_account_tags = organization.get("payerAccount", {}).get(
+                "organizationAccountTags", {}
+            )
+            if isinstance(payer_account_tags, str):
+                payer_account_tags = json.loads(payer_account_tags)
+
+            account_tags = organization.get("tags", {})
+            if isinstance(account_tags, str):
+                account_tags = json.loads(account_tags)
+
+    return payer_account_tags | account_tags

reconcile/typed_queries/saas_files.py

@@ -1,5 +1,4 @@
 import hashlib
-import json
 from collections.abc import Callable
 from threading import Lock
 from typing import Any
@@ -48,6 +47,7 @@ from reconcile.utils.exceptions import (
     AppInterfaceSettingsError,
     ParameterError,
 )
+from reconcile.utils.json import json_dumps
 from reconcile.utils.jsonpath import parse_jsonpath
 
 
@@ -302,7 +302,7 @@ def convert_parameters_to_json_string(root: dict[str, Any]) -> dict[str, Any]:
     """Find all parameter occurrences and convert them to a json string."""
     for key, value in root.items():
         if key in {"parameters", "labels"}:
-            root[key] = json.dumps(value) if value is not None else None
+            root[key] = json_dumps(value) if value is not None else None
         elif isinstance(value, dict):
             root[key] = convert_parameters_to_json_string(value)
         elif isinstance(value, list):

reconcile/utils/aggregated_list.py

@@ -1,8 +1,9 @@
-import json
 import logging
 from collections.abc import Callable, KeysView
 from typing import Any, TypedDict
 
+from reconcile.utils.json import json_dumps
+
 Action = Callable[[Any, list[Any]], bool]
 Cond = Callable[[Any], bool]
 
@@ -89,11 +90,11 @@ class AggregatedList:
         return list(self._dict.values())
 
     def to_json(self) -> str:
-        return json.dumps(self.dump(), indent=4)
+        return json_dumps(self.dump(), indent=4)
 
     @staticmethod
     def hash_params(params: Any) -> int:
-        return hash(json.dumps(params, sort_keys=True))
+        return hash(json_dumps(params))
 
 
 class AggregatedDiffRunner:

reconcile/utils/aws_api.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import operator
 import os
-import re
 from functools import lru_cache
 from threading import Lock
 from typing import (
@@ -25,6 +24,7 @@ import reconcile.utils.lean_terraform_client as terraform
 from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 
 if TYPE_CHECKING:
+    import re
     from collections.abc import (
         Iterable,
         Iterator,
@@ -1074,28 +1074,40 @@ class AWSApi:
         return [rt["RouteTableId"] for rt in vpc_route_tables]
 
     @staticmethod
-    def _filter_amis(
-        images: Iterable[ImageTypeDef], regex: str
-    ) -> list[dict[str, Any]]:
-        results = []
-        pattern = re.compile(regex)
-        for i in images:
-            if not re.search(pattern, i["Name"]):
-                continue
-            if i["State"] != "available":
-                continue
-            item = {"image_id": i["ImageId"], "tags": i.get("Tags", [])}
-            results.append(item)
+    def normalize_tags(tags: Iterable[TagTypeDef]) -> dict[str, str]:
+        return {tag["Key"]: tag["Value"] for tag in tags}
 
-        return results
+    @staticmethod
+    def _filter_amis(
+        images: Iterable[ImageTypeDef],
+        regex: re.Pattern,
+    ) -> dict[str, dict[str, str]]:
+        return {
+            image["ImageId"]: AWSApi.normalize_tags(image.get("Tags", []))
+            for image in images
+            if regex.search(image["Name"]) and image["State"] == "available"
+        }
 
     def get_amis_details(
         self,
         account: Mapping[str, Any],
         owner_account: Mapping[str, Any],
-        regex: str,
+        regex: re.Pattern,
         region: str | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> dict[str, dict[str, str]]:
+        """
+        Get AMI details for an account, find AMI name matches regex and state is available.
+        Return ImageId and normalized tags.
+
+        Args:
+            account: AWS account
+            owner_account: AMI owner AWS account uid
+            regex: regex to filter AMI name
+            region: AWS account region
+
+        Returns:
+            dict[str, dict[str, str]]: Key is AMI ImageId, value is AMI normalized tags.
+        """
         ec2 = self._account_ec2_client(account["name"], region_name=region)
         images = self.get_account_amis(ec2, owner=owner_account["uid"])
         return self._filter_amis(images, regex)
@@ -1175,12 +1187,31 @@ class AWSApi:
         client = self._account_cloudwatch_client(account_name, region_name=region_name)
         client.delete_log_group(logGroupName=group_name)
 
-    def create_tag(
-        self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
+    def create_tags(
+        self,
+        account: Mapping[str, Any],
+        resource_id: str,
+        tags: Mapping[str, str],
     ) -> None:
+        """
+        Create tags on EC2 resources (AMI)
+
+        Args:
+            account: AWS account
+            resource_id: AWS resource id
+            tags: tags to update
+
+        Returns:
+            None
+        """
         ec2 = self._account_ec2_client(account["name"])
-        tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
-        ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
+        formatted_tags: list[TagTypeDef] = [
+            {"Key": k, "Value": v} for k, v in tags.items()
+        ]
+        ec2.create_tags(
+            Resources=[resource_id],
+            Tags=formatted_tags,
+        )
 
     def get_alb_network_interface_ips(
         self, account: awsh.Account, service_name: str