qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/terrascript_aws_client.py

@@ -1,6 +1,5 @@
 import base64
 import enum
-import imghdr
 import json
 import logging
 import os
@@ -8,12 +7,9 @@ import random
 import re
 import string
 import tempfile
-from collections.abc import (
-    Iterable,
-    Mapping,
-    MutableMapping,
-)
-from dataclasses import dataclass
+from collections import Counter
+from collections.abc import Iterable, Mapping, MutableMapping
+from dataclasses import dataclass, field
 from ipaddress import (
     ip_address,
     ip_network,
@@ -22,11 +18,11 @@ from json import JSONDecodeError
 from threading import Lock
 from typing import (
     Any,
-    Optional,
     cast,
 )
 
 import anymarkup
+import filetype
 import requests
 from botocore.errorfactory import ClientError
 from github import Github
@@ -35,7 +31,9 @@ from sretoolbox.utils import threaded
 # temporary to create aws_ecrpublic_repository
 from terrascript import (
     Backend,
+    Block,
     Data,
+    Module,
     Output,
     Provider,
     Resource,
@@ -90,11 +88,11 @@ from terrascript.resource import (
     aws_iam_role,
     aws_iam_role_policy,
     aws_iam_role_policy_attachment,
+    aws_iam_saml_provider,
     aws_iam_service_linked_role,
     aws_iam_user,
     aws_iam_user_group_membership,
     aws_iam_user_login_profile,
-    aws_iam_user_policy,
     aws_iam_user_policy_attachment,
     aws_kinesis_stream,
     aws_kms_alias,
@@ -108,6 +106,8 @@ from terrascript.resource import (
     aws_lb_listener_rule,
     aws_lb_target_group,
     aws_lb_target_group_attachment,
+    aws_msk_cluster,
+    aws_msk_configuration,
     aws_ram_principal_association,
     aws_ram_resource_association,
     aws_ram_resource_share,
@@ -138,17 +138,27 @@ from terrascript.resource import (
     random_id,
 )
 
-import reconcile.openshift_resources_base as orb
 import reconcile.utils.aws_helper as awsh
 from reconcile import queries
+from reconcile.cli import TERRAFORM_VERSION
 from reconcile.github_org import get_default_config
+from reconcile.gql_definitions.fragments.aws_vpc_request import (
+    VPCRequest,
+)
+from reconcile.gql_definitions.terraform_resources.terraform_resources_namespaces import (
+    NamespaceTerraformResourceLifecycleV1,
+)
 from reconcile.utils import gql
 from reconcile.utils.aws_api import (
     AmiTag,
     AWSApi,
 )
+from reconcile.utils.cloud_resource_best_practice.aws_rds import (
+    verify_rds_best_practices,
+)
 from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.elasticsearch_exceptions import (
+    ElasticSearchResourceColdStorageError,
     ElasticSearchResourceMissingSubnetIdError,
     ElasticSearchResourceNameInvalidError,
     ElasticSearchResourceZoneAwareSubnetInvalidError,
@@ -168,12 +178,13 @@ from reconcile.utils.external_resources import (
 from reconcile.utils.git import is_file_in_git_repo
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi
+from reconcile.utils.jinja2.utils import process_extracurlyjinja2_template
 from reconcile.utils.ocm import OCMMap
 from reconcile.utils.password_validator import (
     PasswordPolicy,
     PasswordValidator,
 )
-from reconcile.utils.secret_reader import SecretReader
+from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
 from reconcile.utils.terraform import safe_resource_id
 
 GH_BASE_URL = os.environ.get("GITHUB_API", "https://api.github.com")
@@ -217,7 +228,9 @@ VARIABLE_KEYS = [
     "image",
     "assume_role",
     "inline_policy",
+    "role_policy",
     "assume_condition",
+    "assume_action",
     "api_proxy_uri",
     "cognito_callback_bucket_name",
     "openshift_ingress_load_balancer_arn",
@@ -232,6 +245,7 @@ VARIABLE_KEYS = [
     "subscriptions",
     "records",
     "extra_tags",
+    "lifecycle",
 ]
 
 EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")
@@ -250,6 +264,28 @@ SUPPORTED_ALB_LISTENER_RULE_CONDITION_TYPE_MAPPING = {
     "source-ip": "source_ip",
 }
 
+DEFAULT_TAGS = {
+    "tags": {
+        "app": "app-sre-infra",
+    },
+}
+
+
+class OutputResourceNameNotUniqueException(Exception):
+    def __init__(self, namespace, duplicates):
+        self.namespace, self.duplicates = namespace, duplicates
+        super().__init__(
+            str.format(
+                "Found duplicate values {} for 'output_resource_name' in namespace {}. Please ensure 'output_resource_name' is unique in a given namespace.",
+                self.duplicates,
+                self.namespace,
+            )
+        )
+
+
+class RDSParameterGroupValidationError(Exception):
+    pass
+
 
 class StateInaccessibleException(Exception):
     pass
@@ -264,6 +300,13 @@ class UnapprovedSecretPathError(Exception):
     pass
 
 
+class Moved(Block):
+    """Terraform `moved` block, available since Terraform 1.1"""
+
+    def __init__(self, fro: str, to: str):
+        super().__init__(fro=fro, to=to)
+
+
 class aws_ecrpublic_repository(Resource):
     pass
 
@@ -272,10 +315,18 @@ class aws_s3_bucket_acl(Resource):
     pass
 
 
+class aws_s3_bucket_logging(Resource):
+    pass
+
+
 class aws_cloudfront_log_delivery_canonical_user_id(Data):
     pass
 
 
+class cloudinit_config(Data):
+    pass
+
+
 # temporary until we upgrade to a terrascript release
 # that supports this provider
 # https://github.com/mjuenema/python-terrascript/pull/166
@@ -302,6 +353,18 @@ class aws_api_gateway_rest_api_policy(Resource):
     pass
 
 
+# temporary until we upgrade to a terrascript release
+# that supports this resource
+class aws_msk_scram_secret_association(Resource):
+    pass
+
+
+# temporary until we upgrade to a terrascript release
+# that supports this resource
+class aws_secretsmanager_secret_policy(Resource):
+    pass
+
+
 class ElasticSearchLogGroupType(enum.Enum):
     INDEX_SLOW_LOGS = "INDEX_SLOW_LOGS"
     SEARCH_SLOW_LOGS = "SEARCH_SLOW_LOGS"
@@ -316,6 +379,22 @@ class ElasticSearchLogGroupInfo:
     log_group_identifier: str
 
 
+@dataclass
+class Exclusion:
+    all: bool = False
+    provisioners: set[str] = field(default_factory=set)
+
+
+class ProviderExcludedError(Exception):
+    def __init__(self, spec: ExternalResourceSpec) -> None:
+        super().__init__(
+            self,
+            "The provider is not managed by terraform_resources in this provisioner. "
+            "Set the `managed_by_erv2: true` attribute in the external resource spec to fix it."
+            f"Provisioner: {spec.provisioner['name']}, Provider: {spec.provider}, Identifier: {spec.resource['identifier']}",
+        )
+
+
 class TerrascriptClient:  # pylint: disable=too-many-public-methods
     """
     At a high-level, this class is responsible for generating Terraform configuration in
@@ -337,17 +416,23 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         integration_prefix: str,
         thread_pool_size: int,
         accounts: Iterable[dict[str, Any]],
-        settings: Optional[Mapping[str, Any]] = None,
-        prefetch_resources_by_schemas: Optional[list[str]] = None,
+        settings: Mapping[str, Any] | None = None,
+        prefetch_resources_by_schemas: list[str] | None = None,
+        secret_reader: SecretReaderBase | None = None,
     ) -> None:
         self.integration = integration
         self.integration_prefix = integration_prefix
-        self.settings = settings
         self.thread_pool_size = thread_pool_size
         filtered_accounts = self.filter_disabled_accounts(accounts)
-        self.secret_reader = SecretReader(settings=settings)
+        if secret_reader:
+            self.secret_reader = secret_reader
+        else:
+            self.secret_reader = SecretReader(settings=settings)
+        self.configs: dict[str, dict] = {}
         self.populate_configs(filtered_accounts)
-        self.versions = {a["name"]: a["providerVersion"] for a in filtered_accounts}
+        self.versions: dict[str, str] = {
+            a["name"]: a["providerVersion"] for a in filtered_accounts
+        }
         tss = {}
         locks = {}
         self.supported_regions = {}
@@ -361,32 +446,47 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     ts += provider.aws(
                         access_key=config["aws_access_key_id"],
                         secret_key=config["aws_secret_access_key"],
-                        version=self.versions.get(name),
                         region=region,
                         alias=region,
+                        skip_region_validation=True,
+                        default_tags=DEFAULT_TAGS,
                     )
 
             # Add default region, which will be in resourcesDefaultRegion
             ts += provider.aws(
                 access_key=config["aws_access_key_id"],
                 secret_key=config["aws_secret_access_key"],
-                version=self.versions.get(name),
                 region=config["resourcesDefaultRegion"],
+                skip_region_validation=True,
+                default_tags=DEFAULT_TAGS,
             )
 
-            # the time provider can be removed if all AWS accounts
-            # upgrade to a provider version with this bug fix
-            # https://github.com/hashicorp/terraform-provider-aws/pull/20926
-            ts += time(version="0.9.1")
-
-            ts += provider.random(version="3.4.3")
-
-            ts += provider.template(version="2.2.0")
-
             ts += Terraform(
                 backend=TerrascriptClient.state_bucket_for_account(
                     self.integration, name, config
-                )
+                ),
+                required_providers={
+                    "aws": {
+                        "source": "hashicorp/aws",
+                        "version": self.versions.get(name),
+                    },
+                    # the time provider can be removed if all AWS accounts
+                    # upgrade to a provider version with this bug fix
+                    # https://github.com/hashicorp/terraform-provider-aws/pull/20926
+                    "time": {
+                        "source": "hashicorp/time",
+                        "version": "0.9.1",
+                    },
+                    "random": {
+                        "source": "hashicorp/random",
+                        "version": "3.4.3",
+                    },
+                    "cloudinit": {
+                        "source": "hashicorp/cloudinit",
+                        "version": "2.3.3",
+                    },
+                },
+                required_version=TERRAFORM_VERSION[0],
             )
             tss[name] = ts
             locks[name] = Lock()
@@ -402,8 +502,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         self.accounts = {a["name"]: a for a in filtered_accounts}
         self.uids = {a["name"]: a["uid"] for a in filtered_accounts}
+        # default_regions info is needed in populate_tf_resource_rds, even in disabled accounts
         self.default_regions = {
-            a["name"]: a["resourcesDefaultRegion"] for a in filtered_accounts
+            a["name"]: a["resourcesDefaultRegion"] for a in accounts
         }
         self.partitions = {
             a["name"]: a.get("partition") or "aws" for a in filtered_accounts
@@ -414,9 +515,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         self.rosa_authenticator_pre_signup_zip_lock = Lock()
         self.lambda_zip: dict[str, str] = {}
         self.lambda_lock = Lock()
-        self.github: Optional[Github] = None
+        self.github: Github | None = None
         self.github_lock = Lock()
-        self.gitlab: Optional[GitLabApi] = None
+        self.gitlab: GitLabApi | None = None
         self.gitlab_lock = Lock()
         self.jenkins_map: dict[str, JenkinsApi] = {}
         self.jenkins_lock = Lock()
@@ -445,7 +546,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # defaults from account
         bucket_backend_value = config.get("bucket")
-        key_backend_value = config.get("{}_key".format(integration))
+        key_backend_value = config.get(f"{integration}_key")
         region_backend_value = config.get("region")
         terraform_state = config["terraformState"]
         if terraform_state:
@@ -567,7 +668,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             with self.gitlab_lock:
                 if not self.gitlab:
                     instance = queries.get_gitlab_instance()
-                    self.gitlab = GitLabApi(instance, settings=self.settings)
+                    self.gitlab = GitLabApi(instance, secret_reader=self.secret_reader)
         return self.gitlab
 
     def init_jenkins(self, instance: dict) -> JenkinsApi:
@@ -576,10 +677,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             with self.jenkins_lock:
                 # this may have already happened, so we check again
                 if not self.jenkins_map.get(instance_name):
-                    self.jenkins_map[
-                        instance_name
-                    ] = JenkinsApi.init_jenkins_from_secret(
-                        SecretReader(self.settings), instance["token"]
+                    self.jenkins_map[instance_name] = (
+                        JenkinsApi.init_jenkins_from_secret(
+                            self.secret_reader, instance["token"]
+                        )
                     )
         return self.jenkins_map[instance_name]
 
@@ -600,7 +701,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.thread_pool_size,
             secret_reader=self.secret_reader,
         )
-        self.configs: dict[str, dict] = {}
         for account_name, config in results:
             account = awsh.get_account(accounts, account_name)
             config["supportedDeploymentRegions"] = account["supportedDeploymentRegions"]
@@ -635,6 +735,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 group_name = aws_group["name"]
                 group_policies = aws_group["policies"]
                 account = aws_group["account"]
+                if account["sso"] is True:
+                    # AWS accounts with SSO enabled do not need IAM groups
+                    continue
                 account_name = account["name"]
                 if account_name not in groups:
                     groups[account_name] = {}
@@ -691,7 +794,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         self,
         roles,
         skip_reencrypt_accounts: list[str],
-        appsre_pgp_key: Optional[str],
+        appsre_pgp_key: str | None,
     ):
         error = False
         for role in roles:
@@ -705,6 +808,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             for aws_group in aws_groups:
                 group_name = aws_group["name"]
                 account = aws_group["account"]
+                if account["sso"] is True:
+                    # AWS accounts with SSO enabled do not need IAM users
+                    continue
                 account_name = account["name"]
                 account_console_url = account["consoleUrl"]
 
@@ -716,11 +822,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
                 # we want to include the console url in the outputs
                 # to be used later to generate the email invitations
-                output_name_0_13 = "{}_console-urls__{}".format(
-                    self.integration_prefix, account_name
-                )
+                output_name = f"{self.integration_prefix}_console-urls__{account_name}"
                 output_value = account_console_url
-                tf_output = Output(output_name_0_13, value=output_value)
+                tf_output = Output(output_name, value=output_value)
                 self.add_resource(account_name, tf_output)
 
                 for user in users:
@@ -777,25 +881,23 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     # we want the outputs to be formed into a mail invitation
                     # for each new user. we form an output of the form
                     # 'qrtf.enc-passwords[user_name] = <encrypted password>
-                    output_name_0_13 = "{}_enc-passwords__{}".format(
-                        self.integration_prefix, user_name
+                    output_name = (
+                        f"{self.integration_prefix}_enc-passwords__{user_name}"
                     )
                     output_value = (
                         "${" + tf_iam_user_login_profile.encrypted_password + "}"
                     )
-                    tf_output = Output(output_name_0_13, value=output_value)
+                    tf_output = Output(output_name, value=output_value, sensitive=True)
                     self.add_resource(account_name, tf_output)
 
             for user_policy in user_policies:
+                if user_policy["account"]["sso"] is True:
+                    # AWS accounts with SSO enabled do not need user policies
+                    continue
                 policy_name = user_policy["name"]
                 account_name = user_policy["account"]["name"]
-                account_uid = user_policy["account"]["uid"]
                 for user in users:
-                    # replace known keys with values
                     user_name = self._get_aws_username(user)
-                    policy = user_policy["policy"]
-                    policy = policy.replace("${aws:username}", user_name)
-                    policy = policy.replace("${aws:accountid}", account_uid)
 
                     # Ref: terraform aws_iam_policy
                     tf_iam_user = self.get_tf_iam_user(user_name)
@@ -803,7 +905,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     tf_aws_iam_policy = aws_iam_policy(
                         identifier,
                         name=identifier,
-                        policy=policy,
+                        policy=user_policy["policy"],
                     )
                     self.add_resource(account_name, tf_aws_iam_policy)
                     # Ref: terraform aws_iam_user_policy_attachment
@@ -811,9 +913,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         identifier,
                         user=user_name,
                         policy_arn=f"${{{tf_aws_iam_policy.arn}}}",
-                        depends_on=self.get_dependencies(
-                            [tf_iam_user, tf_aws_iam_policy]
-                        ),
+                        depends_on=self.get_dependencies([
+                            tf_iam_user,
+                            tf_aws_iam_policy,
+                        ]),
                     )
                     self.add_resource(account_name, tf_iam_user_policy_attachment)
 
@@ -823,7 +926,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         self,
         roles,
         skip_reencrypt_accounts: list[str],
-        appsre_pgp_key: Optional[str] = None,
+        appsre_pgp_key: str | None = None,
     ):
         self.populate_iam_groups(roles)
         err = self.populate_iam_users(
@@ -834,27 +937,59 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         return err
 
     @staticmethod
-    def get_alias_name_from_assume_role(assume_role):
-        uid = awsh.get_account_uid_from_arn(assume_role)
-        return f"account-{uid}"
+    def get_provider_alias(account: Mapping[str, str]) -> str:
+        if not account.get("assume_role"):
+            return f"account-{account['name']}-{account['assume_region']}"
+        uid = awsh.get_account_uid_from_arn(account["assume_role"])
+        role_name = awsh.get_id_from_arn(account["assume_role"])
+        return f"account-{uid}-{role_name}"
 
-    def populate_additional_providers(self, accounts):
+    @staticmethod
+    def get_resource_lifecycle(
+        common_values: dict[str, Any],
+    ) -> dict[str, Any] | None:
+        if lifecycle := common_values.get("lifecycle"):
+            lifecycle = NamespaceTerraformResourceLifecycleV1(**lifecycle)
+            if lifecycle.create_before_destroy is None:
+                lifecycle.create_before_destroy = False
+            if lifecycle.prevent_destroy is None:
+                lifecycle.prevent_destroy = False
+            if lifecycle.ignore_changes is None:
+                lifecycle.ignore_changes = []
+            if "all" in lifecycle.ignore_changes:
+                lifecycle.ignore_changes = "all"
+            return lifecycle.dict(by_alias=True)
+        return None
+
+    def populate_additional_providers(self, infra_account_name: str, accounts):
         for account in accounts:
             account_name = account["name"]
-            assume_role = account["assume_role"]
-            alias = self.get_alias_name_from_assume_role(assume_role)
-            ts = self.tss[account_name]
+            assume_role = account.get("assume_role")
+            region = account["assume_region"]
+            alias = self.get_provider_alias(account)
+            ts = self.tss[infra_account_name]
             config = self.configs[account_name]
             existing_provider_aliases = {p.get("alias") for p in ts["provider"]["aws"]}
             if alias not in existing_provider_aliases:
-                ts += provider.aws(
-                    access_key=config["aws_access_key_id"],
-                    secret_key=config["aws_secret_access_key"],
-                    version=self.versions.get(account_name),
-                    region=account["assume_region"],
-                    alias=alias,
-                    assume_role={"role_arn": assume_role},
-                )
+                if assume_role:
+                    ts += provider.aws(
+                        access_key=config["aws_access_key_id"],
+                        secret_key=config["aws_secret_access_key"],
+                        region=region,
+                        alias=alias,
+                        assume_role={"role_arn": assume_role},
+                        skip_region_validation=True,
+                        default_tags=DEFAULT_TAGS,
+                    )
+                else:
+                    ts += provider.aws(
+                        access_key=config["aws_access_key_id"],
+                        secret_key=config["aws_secret_access_key"],
+                        region=region,
+                        alias=alias,
+                        skip_region_validation=True,
+                        default_tags=DEFAULT_TAGS,
+                    )
 
     def populate_route53(
         self, desired_state: Iterable[dict[str, Any]], default_ttl: int = 300
@@ -913,7 +1048,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 record["health_check_id"] = f"${{{healthcheck_resource.id}}}"
 
             # Get value from Vault if _records_from_vault was set
-            records_from_vault: Optional[Iterable[dict[str, str]]] = record.pop(
+            records_from_vault: Iterable[dict[str, str]] | None = record.pop(
                 "records_from_vault", None
             )
             if records_from_vault:
@@ -922,19 +1057,17 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 )
                 vault_values: list[str] = []
                 for rec in records_from_vault:
-                    if not rec["path"] in allowed_vault_secret_paths:
+                    if rec["path"] not in allowed_vault_secret_paths:
                         raise UnapprovedSecretPathError(
                             "'{}' is not in the list of approved Vault secret paths. Add this path to 'allowed_vault_secret_paths'.".format(
                                 rec["path"]
                             )
                         )
-                    value = self.secret_reader.read(
-                        {
-                            "path": rec["path"],
-                            "field": rec["field"],
-                            "version": rec["version"],
-                        }
-                    )
+                    value = self.secret_reader.read({
+                        "path": rec["path"],
+                        "field": rec["field"],
+                        "version": rec["version"],
+                    })
                     # 'key' is only set when the secret data is in JSON format and a
                     # specific item from the object needs to be selected, otherwise the
                     # value is used as-is.
@@ -950,7 +1083,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         except KeyError:
                             msg = f"Key '{rec['key']}' was not found in the contents of secret '{rec['path']}'"
                             logging.error(msg)
-                            raise KeyError(msg)
+                            raise KeyError(msg) from None
                     vault_values.append(value)
                 record["records"] = vault_values
 
@@ -963,14 +1096,19 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             if item["deleted"]:
                 continue
 
+            infra_account_name = item["infra_account_name"]
+
             connection_provider = item["connection_provider"]
             connection_name = item["connection_name"]
             requester = item["requester"]
             accepter = item["accepter"]
 
             req_account = requester["account"]
-            req_account_name = req_account["name"]
-            req_alias = self.get_alias_name_from_assume_role(req_account["assume_role"])
+            req_alias = self.get_provider_alias(req_account)
+
+            acc_account = accepter["account"]
+            acc_account_name = acc_account["name"]
+            acc_alias = self.get_provider_alias(acc_account)
 
             # Requester's side of the connection - the cluster's account
             identifier = f"{requester['vpc_id']}-{accepter['vpc_id']}"
@@ -981,7 +1119,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "vpc_id": requester["vpc_id"],
                 "peer_vpc_id": accepter["vpc_id"],
                 "peer_region": accepter["region"],
-                "peer_owner_id": req_account["uid"],
+                "peer_owner_id": acc_account["uid"],
                 "auto_accept": False,
                 "tags": {
                     "managed_by_integration": self.integration,
@@ -993,7 +1131,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             if req_peer_owner_id:
                 values["peer_owner_id"] = req_peer_owner_id
             tf_resource = aws_vpc_peering_connection(identifier, **values)
-            self.add_resource(req_account_name, tf_resource)
+            self.add_resource(infra_account_name, tf_resource)
 
             # add routes to existing route tables
             route_table_ids = requester.get("route_table_ids")
@@ -1009,11 +1147,38 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     }
                     route_identifier = f"{identifier}-{route_table_id}"
                     tf_resource = aws_route(route_identifier, **values)
-                    self.add_resource(req_account_name, tf_resource)
-
-            acc_account = accepter["account"]
-            acc_account_name = acc_account["name"]
-            acc_alias = self.get_alias_name_from_assume_role(acc_account["assume_role"])
+                    self.add_resource(infra_account_name, tf_resource)
+
+            # add security group rules for private hosted controlplane API VPC endpoint service
+            if requester.get("api_security_group_id"):
+                hcp_api_ingress_rule = aws_security_group_rule(
+                    f"requester-api-access-from-peering-{connection_name}",
+                    provider="aws." + req_alias,
+                    type="ingress",
+                    security_group_id=requester.get("api_security_group_id"),
+                    cidr_blocks=[accepter["cidr_block"]],
+                    from_port=443,
+                    to_port=443,
+                    protocol="tcp",
+                    description=f"HCP API access from peering connection {connection_name}",
+                )
+                self.add_resource(infra_account_name, hcp_api_ingress_rule)
+
+            if accepter.get("api_security_group_id"):
+                self.add_resource(
+                    infra_account_name,
+                    aws_security_group_rule(
+                        f"accepter-api-access-from-peering-{connection_name}",
+                        provider="aws." + acc_alias,
+                        type="ingress",
+                        security_group_id=accepter.get("api_security_group_id"),
+                        cidr_blocks=[requester["cidr_block"]],
+                        from_port=443,
+                        to_port=443,
+                        protocol="tcp",
+                        description=f"HCP API access from peering connection {connection_name}",
+                    ),
+                )
 
             # Accepter's side of the connection.
             values = {
@@ -1027,13 +1192,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     "Name": connection_name,
                 },
             }
-            if connection_provider in ["account-vpc", "account-vpc-mesh"]:
+            if connection_provider in {"account-vpc", "account-vpc-mesh"}:
                 if self._multiregion_account(acc_account_name):
                     values["provider"] = "aws." + accepter["region"]
             else:
                 values["provider"] = "aws." + acc_alias
             tf_resource = aws_vpc_peering_connection_accepter(identifier, **values)
-            self.add_resource(acc_account_name, tf_resource)
+            self.add_resource(infra_account_name, tf_resource)
 
             # add routes to existing route tables
             route_table_ids = accepter.get("route_table_ids")
@@ -1046,32 +1211,121 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         + identifier
                         + ".id}",
                     }
-                    if connection_provider in ["account-vpc", "account-vpc-mesh"]:
+                    if connection_provider in {"account-vpc", "account-vpc-mesh"}:
                         if self._multiregion_account(acc_account_name):
                             values["provider"] = "aws." + accepter["region"]
                     else:
                         values["provider"] = "aws." + acc_alias
                     route_identifier = f"{identifier}-{route_table_id}"
                     tf_resource = aws_route(route_identifier, **values)
-                    self.add_resource(acc_account_name, tf_resource)
+                    self.add_resource(infra_account_name, tf_resource)
+
+    def populate_vpc_requests(
+        self, vpc_requests: Iterable[VPCRequest], vpc_module_version: str
+    ) -> None:
+        for request in vpc_requests:
+            # skiping deleted requests
+            if request.delete:
+                continue
+            # The default values here come from infra repo's module configuration
+            vpc_module_values = {
+                "source": "terraform-aws-modules/vpc/aws",
+                "version": vpc_module_version,
+                "name": request.identifier,
+                "cidr": request.cidr_block.network_address,
+                "private_subnet_tags": {"kubernetes.io/role/internal-elb": "1"},
+                "public_subnet_tags": {"kubernetes.io/role/elb": "1"},
+                "create_database_subnet_group": False,
+                "enable_dns_hostnames": True,
+                "tags": {
+                    "managed_by_integration": self.integration,
+                },
+            }
+
+            if request.subnets and request.subnets.public:
+                vpc_module_values["public_subnets"] = request.subnets.public
+            if request.subnets and request.subnets.private:
+                vpc_module_values["private_subnets"] = request.subnets.private
+            if request.subnets and request.subnets.availability_zones:
+                vpc_module_values["azs"] = request.subnets.availability_zones
+
+            # We only want to enable nat_gateway if we have public and private subnets
+            if request.subnets and request.subnets.public and request.subnets.private:
+                vpc_module_values["enable_nat_gateway"] = True
+
+            aws_account = request.account.name
+            vpc_module = Module(request.identifier, **vpc_module_values)
+            self.add_resource(aws_account, vpc_module)
+
+            # vpc_endpoint
+            vpc_endpoint_values = {
+                "source": "terraform-aws-modules/vpc/aws//modules/vpc-endpoints",
+                "version": vpc_module_version,
+                "vpc_id": f"${{{vpc_module.vpc_id}}}",
+                "endpoints": {
+                    "s3": {
+                        "service": "s3",
+                        "serivce_type": "Gateway",
+                        "tags": {
+                            "managed_by_integration": self.integration,
+                            "Name": f"{request.identifier}--vpce-s3",
+                            "route_table_ids": vpc_module.vpc.private_route_table_ids,
+                        },
+                    }
+                },
+            }
+
+            vpc_endpoint = Module(
+                f"{request.identifier}-endpoint", **vpc_endpoint_values
+            )
+            self.add_resource(aws_account, vpc_endpoint)
+
+            # The outputs for module are only working with this sintaxe
+            vpc_id_output = Output(
+                f"{request.identifier}-vpc_id", value=f"${{{vpc_module.vpc_id}}}"
+            )
+            self.add_resource(aws_account, vpc_id_output)
+
+            vpc_cidr_block_output = Output(
+                f"{request.identifier}-vpc_cidr_block",
+                value=f"${{{vpc_module.vpc_cidr_block}}}",
+            )
+            self.add_resource(aws_account, vpc_cidr_block_output)
+
+            if request.subnets and request.subnets.private:
+                private_subnets_output = Output(
+                    f"{request.identifier}-private_subnets",
+                    value=f"${{module.{request.identifier}.private_subnets}}",
+                )
+                self.add_resource(aws_account, private_subnets_output)
+
+            if request.subnets and request.subnets.public:
+                public_subnets_output = Output(
+                    f"{request.identifier}-public_subnets",
+                    value=f"${{module.{request.identifier}.public_subnets}}",
+                )
+                self.add_resource(aws_account, public_subnets_output)
 
     def populate_tgw_attachments(self, desired_state):
         for item in desired_state:
-            if item["deleted"]:
+            if item.deleted:
                 continue
 
-            connection_name = item["connection_name"]
-            requester = item["requester"]
-            accepter = item["accepter"]
+            infra_account_name = item.infra_acount_name
+
+            connection_name = item.connection_name
+            requester = item.requester
+            accepter = item.accepter
 
             # Requester's side of the connection - the AWS account
-            req_account = requester["account"]
-            req_account_name = req_account["name"]
+            req_account = requester.account
+            req_account_name = req_account.name
             # Accepter's side of the connection - the cluster's account
-            acc_account = accepter["account"]
-            acc_account_name = acc_account["name"]
-            acc_alias = self.get_alias_name_from_assume_role(acc_account["assume_role"])
-            acc_uid = awsh.get_account_uid_from_arn(acc_account["assume_role"])
+            acc_account = accepter.account
+            acc_alias = self.get_provider_alias(acc_account.dict(by_alias=True))
+            acc_uid = acc_account.uid
+            if acc_account.assume_role:
+                acc_uid = awsh.get_account_uid_from_arn(acc_account.assume_role)
 
             tags = {"managed_by_integration": self.integration, "Name": connection_name}
             # add resource share
@@ -1081,9 +1335,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "tags": tags,
             }
             if self._multiregion_account(req_account_name):
-                values["provider"] = "aws." + requester["region"]
+                values["provider"] = "aws." + requester.region
             tf_resource_share = aws_ram_resource_share(connection_name, **values)
-            self.add_resource(req_account_name, tf_resource_share)
+            self.add_resource(infra_account_name, tf_resource_share)
 
             # share with accepter aws account
             values = {
@@ -1091,11 +1345,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "resource_share_arn": "${" + tf_resource_share.arn + "}",
             }
             if self._multiregion_account(req_account_name):
-                values["provider"] = "aws." + requester["region"]
+                values["provider"] = "aws." + requester.region
             tf_resource_association = aws_ram_principal_association(
                 connection_name, **values
             )
-            self.add_resource(req_account_name, tf_resource_association)
+            self.add_resource(infra_account_name, tf_resource_association)
 
             # accept resource share from accepter aws account
             values = {
@@ -1109,32 +1363,32 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             tf_resource_share_accepter = aws_ram_resource_share_accepter(
                 connection_name, **values
             )
-            self.add_resource(acc_account_name, tf_resource_share_accepter)
+            self.add_resource(infra_account_name, tf_resource_share_accepter)
 
             # until now it was standard sharing
             # from this line onwards we will be adding content
             # specific for the TGW attachments integration
 
             # tgw share association
-            identifier = f"{requester['tgw_id']}-{accepter['vpc_id']}"
+            identifier = f"{requester.tgw_id}-{accepter.vpc_id}"
             values = {
-                "resource_arn": requester["tgw_arn"],
+                "resource_arn": requester.tgw_arn,
                 "resource_share_arn": "${" + tf_resource_share.arn + "}",
             }
             if self._multiregion_account(req_account_name):
-                values["provider"] = "aws." + requester["region"]
+                values["provider"] = "aws." + requester.region
             tf_resource_association = aws_ram_resource_association(identifier, **values)
-            self.add_resource(req_account_name, tf_resource_association)
+            self.add_resource(infra_account_name, tf_resource_association)
 
             # now that the tgw is shared to the cluster's aws account
             # we can create a vpc attachment to the tgw
-            subnets_id_az = accepter["subnets_id_az"]
+            subnets_id_az = accepter.subnets_id_az
             subnets = self.get_az_unique_subnet_ids(subnets_id_az)
             values = {
                 "provider": "aws." + acc_alias,
                 "subnet_ids": subnets,
-                "transit_gateway_id": requester["tgw_id"],
-                "vpc_id": accepter["vpc_id"],
+                "transit_gateway_id": requester.tgw_id,
+                "vpc_id": accepter.vpc_id,
                 "depends_on": [
                     "aws_ram_principal_association." + connection_name,
                     "aws_ram_resource_association." + identifier,
@@ -1145,7 +1399,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 identifier, **values
             )
             # we send the attachment from the cluster's aws account
-            self.add_resource(acc_account_name, tf_resource_attachment)
+            self.add_resource(infra_account_name, tf_resource_attachment)
 
             # and accept the attachment in the non cluster's aws account
             values = {
@@ -1153,30 +1407,45 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "tags": tags,
             }
             if self._multiregion_account(req_account_name):
-                values["provider"] = "aws." + requester["region"]
+                values["provider"] = "aws." + requester.region
             tf_resource_attachment_accepter = (
                 aws_ec2_transit_gateway_vpc_attachment_accepter(identifier, **values)
             )
-            self.add_resource(req_account_name, tf_resource_attachment_accepter)
+            self.add_resource(infra_account_name, tf_resource_attachment_accepter)
 
             # add routes to existing route tables
-            route_table_ids = accepter.get("route_table_ids")
-            req_cidr_block = requester.get("cidr_block")
-            if route_table_ids and req_cidr_block:
+            route_table_ids = accepter.route_table_ids
+            req_cidr_block = requester.cidr_block
+            req_cidr_blocks = requester.cidr_blocks or []
+            if req_cidr_block:
+                req_cidr_blocks.append(req_cidr_block)
+            if route_table_ids and (req_cidr_block or req_cidr_blocks):
                 for route_table_id in route_table_ids:
-                    values = {
-                        "provider": "aws." + acc_alias,
-                        "route_table_id": route_table_id,
-                        "destination_cidr_block": req_cidr_block,
-                        "transit_gateway_id": requester["tgw_id"],
-                    }
-                    route_identifier = f"{identifier}-{route_table_id}"
-                    tf_resource = aws_route(route_identifier, **values)
-                    self.add_resource(acc_account_name, tf_resource)
+                    for cidr_block in req_cidr_blocks:
+                        values = {
+                            "provider": "aws." + acc_alias,
+                            "route_table_id": route_table_id,
+                            "destination_cidr_block": cidr_block,
+                            "transit_gateway_id": requester.tgw_id,
+                        }
+                        # use the cidr block in the resource name to allow re-ordering
+                        cidr_id = cidr_block.replace(".", "-").replace("/", "_")
+                        route_identifier = (
+                            f"{identifier}-{route_table_id}-dest-{cidr_id}"
+                        )
+                        tf_resource = aws_route(route_identifier, **values)
+                        self.add_resource(infra_account_name, tf_resource)
+                    if req_cidr_block:
+                        req_cidr_id = req_cidr_block.replace(".", "-").replace("/", "_")
+                        moved = Moved(
+                            fro=f"aws_route.{identifier}-{route_table_id}",
+                            to=f"aws_route.{identifier}-{route_table_id}-dest-{req_cidr_id}",
+                        )
+                        self.add_moved(infra_account_name, moved)
 
             # add routes to peered transit gateways in the requester's
             # account to achieve global routing from all regions
-            requester_routes = requester.get("routes")
+            requester_routes = requester.routes
             if requester_routes:
                 for route in requester_routes:
                     route_region = route["region"]
@@ -1197,11 +1466,25 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     tf_resource = aws_ec2_transit_gateway_route(
                         route_identifier, **values
                     )
-                    self.add_resource(req_account_name, tf_resource)
+                    self.add_resource(infra_account_name, tf_resource)
+
+            if accepter.api_security_group_id:
+                hcp_api_ingress_rule = aws_security_group_rule(
+                    f"api-access-from-{requester.tgw_id}",
+                    provider="aws." + acc_alias,
+                    type="ingress",
+                    security_group_id=accepter.api_security_group_id,
+                    cidr_blocks=[requester.cidr_block],
+                    from_port=443,
+                    to_port=443,
+                    protocol="tcp",
+                    description=f"HCP API access from TGW attachment {requester.tgw_id}",
+                )
+                self.add_resource(infra_account_name, hcp_api_ingress_rule)
 
             # add rules to security groups of VPCs which are attached
             # to the transit gateway to allow traffic through the routes
-            requester_rules = requester.get("rules")
+            requester_rules = requester.rules
             if requester_rules:
                 for rule in requester_rules:
                     rule_region = rule["region"]
@@ -1223,25 +1506,25 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         values["provider"] = "aws." + rule_region
                     rule_identifier = f"{identifier}-{rule['vpc_id']}"
                     tf_resource = aws_security_group_rule(rule_identifier, **values)
-                    self.add_resource(req_account_name, tf_resource)
+                    self.add_resource(infra_account_name, tf_resource)
 
-            for zone in requester.get("hostedzones") or []:
+            for zone in requester.hostedzones or []:
                 id = f"{identifier}-{zone}"
                 values = {
-                    "vpc_id": accepter["vpc_id"],
-                    "vpc_region": accepter["region"],
+                    "vpc_id": accepter.vpc_id,
+                    "vpc_region": accepter.region,
                     "zone_id": zone,
                 }
                 authorization = aws_route53_vpc_association_authorization(id, **values)
-                self.add_resource(req_account_name, authorization)
+                self.add_resource(infra_account_name, authorization)
                 values = {
                     "provider": "aws." + acc_alias,
                     "vpc_id": f"${{aws_route53_vpc_association_authorization.{id}.vpc_id}}",
-                    "vpc_region": accepter["region"],
+                    "vpc_region": accepter.region,
                     "zone_id": f"${{aws_route53_vpc_association_authorization.{id}.zone_id}}",
                 }
                 association = aws_route53_zone_association(id, **values)
-                self.add_resource(acc_account_name, association)
+                self.add_resource(infra_account_name, association)
 
     @staticmethod
     def get_az_unique_subnet_ids(subnets_id_az):
@@ -1257,7 +1540,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         return results
 
-    def populate_resources(self, ocm_map: Optional[OCMMap] = None) -> None:
+    def populate_resources(self, ocm_map: OCMMap | None = None) -> None:
         """
         Populates the terraform configuration from resource specs.
         :param ocm_map:
@@ -1266,10 +1549,49 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             for spec in specs:
                 self.populate_tf_resources(spec, ocm_map=ocm_map)
 
+    def _is_provisioner_excluded(
+        self,
+        spec: ExternalResourceSpec,
+        provider_exclusions: Mapping[str, Exclusion],
+    ) -> bool:
+        e = provider_exclusions.get(spec.provider)
+        if not e:
+            return False
+        return e.all or spec.provisioner_name in e.provisioners
+
+    def _filter_specs_managed_by_erv2(
+        self,
+        specs: Iterable[ExternalResourceSpec],
+        provider_exclusions: Mapping[str, Exclusion],
+    ) -> list[ExternalResourceSpec]:
+        filtered_specs = [
+            spec for spec in specs if not spec.resource.get("managed_by_erv2")
+        ]
+
+        for spec in filtered_specs:
+            if self._is_provisioner_excluded(spec, provider_exclusions):
+                raise ProviderExcludedError(spec)
+
+        return filtered_specs
+
+    def _get_provider_exclusions_query_dict(
+        self, provider_exclusions: Iterable[Mapping[str, Any]]
+    ) -> dict[str, Exclusion]:
+        return {
+            item["provider"]: Exclusion(
+                all=item.get("excludeAllProvisioners") or False,
+                provisioners={
+                    p["name"] for p in (item.get("excludeProvisioners") or [])
+                },
+            )
+            for item in provider_exclusions
+        }
+
     def init_populate_specs(
         self,
         namespaces: Iterable[Mapping[str, Any]],
-        account_names: Optional[Iterable[str]],
+        account_names: Iterable[str] | None,
+        provider_exclusions: Iterable[Mapping[str, Any]] | None = None,
     ) -> None:
         """
         Initiates resource specs from the definitions in app-interface
@@ -1280,16 +1602,37 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         self.account_resource_specs: dict[str, list[ExternalResourceSpec]] = {}
         self.resource_spec_inventory: ExternalResourceSpecInventory = {}
 
+        # Ensure provider exclusions are fetched
+        if provider_exclusions is None:
+            provider_exclusions = (
+                queries.get_tf_resources_provider_exclusions_by_provisioner() or []
+            )
+
+        provider_exclusions_query_dict = self._get_provider_exclusions_query_dict(
+            provider_exclusions
+        )
+
         for namespace_info in namespaces:
-            specs = get_external_resource_specs(
-                namespace_info, provision_provider=PROVIDER_AWS
+            all_specs = get_external_resource_specs(
+                namespace_info,
+                provision_provider=PROVIDER_AWS,
+            )
+            specs = self._filter_specs_managed_by_erv2(
+                all_specs, provider_exclusions_query_dict
             )
+            name_counter = Counter(spec.output_resource_name for spec in specs)
+            duplicates = [name for name, count in name_counter.items() if count > 1]
+            if duplicates:
+                raise OutputResourceNameNotUniqueException(
+                    namespace_info.get("name"), duplicates
+                )
             for spec in specs:
                 if account_names and spec.provisioner_name not in account_names:
                     continue
                 self.account_resource_specs.setdefault(
                     spec.provisioner_name, []
                 ).append(spec)
+
                 self.resource_spec_inventory[spec.id_object()] = spec
 
     def populate_tf_resources(self, spec, ocm_map=None):
@@ -1346,6 +1689,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.populate_tf_resource_rosa_authenticator(spec)
         elif provider == "rosa-authenticator-vpce":
             self.populate_tf_resource_rosa_authenticator_vpce(spec)
+        elif provider == "msk":
+            self.populate_tf_resource_msk(spec)
         else:
             raise UnknownProviderError(provider)
 
@@ -1360,12 +1705,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # we want to allow an empty name, so we
         # only validate names which are not empty
-        if values.get("name") and not self.validate_db_name(values["name"]):
+        db_name = self._get_db_name_from_values(values)
+        if db_name and not self.validate_db_name(db_name):
             raise FetchResourceError(
                 f"[{account}] RDS name must contain 1 to 63 letters, "
                 + "numbers, or underscores. RDS name must begin with a "
                 + "letter. Subsequent characters can be letters, "
-                + f"underscores, or digits (0-9): {values['name']}"
+                + f"underscores, or digits (0-9): {db_name}"
             )
 
         if not values.get("apply_immediately"):
@@ -1393,6 +1739,16 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 elif provider != provider_region:
                     raise ValueError("region does not match availability zone")
 
+        def validate_parameter_group(parameter_group: aws_db_parameter_group) -> None:
+            parameter_group_name = parameter_group.get("name")
+            for parameter in parameter_group.get("parameter", []):
+                if parameter.get("name") == "rds.logical_replication":
+                    apply_method = parameter.get("apply_method")
+                    if apply_method != "pending-reboot":
+                        raise RDSParameterGroupValidationError(
+                            f"{parameter_group_name=} rds.logical_replication {apply_method=} must be set to 'pending-reboot'"
+                        )
+
         def populate_parameter_group(name: str) -> aws_db_parameter_group:
             pg_values = self.get_values(name)
             # Parameter group name is not required by terraform.
@@ -1406,7 +1762,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             pg_values["parameter"] = pg_values.pop("parameters")
             if self._multiregion_account(account) and len(provider) > 0:
                 pg_values["provider"] = provider
-            return aws_db_parameter_group(pg_identifier, **pg_values)
+            parameter_group = aws_db_parameter_group(pg_identifier, **pg_values)
+            validate_parameter_group(parameter_group=parameter_group)
+            return parameter_group
 
         # 'deps' should contain a list of terraform resource names
         # (not full objects) that must be created
@@ -1507,9 +1865,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         ca_cert = values.pop("ca_cert", None)
         if ca_cert:
             # db.ca_cert
-            output_name_0_13 = output_prefix + "__db_ca_cert"
+            output_name = output_prefix + "__db_ca_cert"
             output_value = self.secret_reader.read(ca_cert)
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
 
         region = self._region_from_availability_zone(az) or self.default_regions.get(
             account
@@ -1532,6 +1890,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     replica_region = self.default_regions.get(account)
 
                 source_values = self.init_values(source_info)
+                db_name = self._get_db_name_from_values(source_values)
                 if replica_region == region:
                     # replica is in the same region as source
                     values["replicate_source_db"] = replica_source
@@ -1631,7 +1990,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 if sns_topic_name.startswith("arn:"):
                     raise ValueError("destination should not be an arn")
                 e_n["sns_topic"] = "${aws_sns_topic" + "." + sns_topic_name + ".arn}"
-                e_n["source_ids"] = ["${aws_db_instance" + "." + identifier + ".id}"]
+                e_n["source_ids"] = [identifier]
                 source_type = e_n.get("source_type", "all")
                 e_n_identifier = (
                     f"{sns_topic_name}_{source_type}_aws_db_event_subscription"
@@ -1642,33 +2001,34 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         # rds instance
         # Ref: https://www.terraform.io/docs/providers/aws/r/db_instance.html
         tf_resource = aws_db_instance(identifier, **values)
+        verify_rds_best_practices(tf_resource, spec.resource["data_classification"])
         tf_resources.append(tf_resource)
 
         # rds outputs
         # we want the outputs to be formed into an OpenShift Secret
         # with the following fields
         # db.host
-        output_name_0_13 = output_prefix + "__db_host"
+        output_name = output_prefix + "__db_host"
         output_value = "${" + tf_resource.address + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # db.port
-        output_name_0_13 = output_prefix + "__db_port"
+        output_name = output_prefix + "__db_port"
         output_value = "${" + str(tf_resource.port) + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # db.name
-        output_name_0_13 = output_prefix + "__db_name"
-        output_value = output_resource_db_name or values.get("name", "")
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        output_name = output_prefix + "__db_name"
+        output_value = output_resource_db_name or db_name
+        tf_resources.append(Output(output_name, value=output_value))
         # only set db user/password if not a replica or creation from snapshot
         if self._db_needs_auth(values):
             # db.user
-            output_name_0_13 = output_prefix + "__db_user"
+            output_name = output_prefix + "__db_user"
             output_value = values["username"]
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
             # db.password
-            output_name_0_13 = output_prefix + "__db_password"
+            output_name = output_prefix + "__db_password"
             output_value = values["password"]
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
             # only add reset_password key to the terraform state
             # if reset_password_current_value is defined.
             # this means that if the reset_password field is removed
@@ -1676,9 +2036,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             # removed from the state and from the output resource,
             # leading to a recycle of the pods using this resource.
             if reset_password_current_value:
-                output_name_0_13 = output_prefix + "__reset_password"
+                output_name = output_prefix + "__reset_password"
                 output_value = reset_password_current_value
-                tf_resources.append(Output(output_name_0_13, value=output_value))
+                tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -1686,14 +2046,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         if name not in self.configs:
             return False
 
-        if self.configs[name]["supportedDeploymentRegions"] is not None:
-            return True
-
-        return False
+        return self.configs[name]["supportedDeploymentRegions"] is not None
 
     def _find_resource_spec(
         self, account: str, source: str, provider: str
-    ) -> Optional[ExternalResourceSpec]:
+    ) -> ExternalResourceSpec | None:
         if account not in self.account_resource_specs:
             return None
 
@@ -1702,6 +2059,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 return spec
         return None
 
+    def _get_db_name_from_values(self, values: dict) -> str:
+        return values.get("name") or values.get("db_name") or ""
+
     @staticmethod
     def _region_from_availability_zone(az):
         # Find the region by removing the last character from the
@@ -1714,12 +2074,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
     @staticmethod
     def _db_needs_auth(config):
-        if (
+        return bool(
             "replicate_source_db" not in config
             and config.get("replica_source", None) is None
-        ):
-            return True
-        return False
+        )
 
     @staticmethod
     def validate_db_name(name):
@@ -1764,9 +2122,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             common_values.get("server_side_encryption_configuration")
             or DEFAULT_S3_SSE_CONFIGURATION
         )
-        values[
-            "server_side_encryption_configuration"
-        ] = server_side_encryption_configuration
+        values["server_side_encryption_configuration"] = (
+            server_side_encryption_configuration
+        )
         # Support static website hosting [rosa-authenticator]
         website = common_values.get("website")
         if website:
@@ -1819,6 +2177,27 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         if cors_rules:
             # common_values['cors_rules'] is a list of cors_rules
             values["cors_rule"] = cors_rules
+        # S3 Bucket Logging
+        s3_bucket_logging = common_values.get("s3_bucket_logging")
+        if s3_bucket_logging:
+            target_bucket_name = s3_bucket_logging.get("target_bucket_name")
+            logging_identifier = f"{identifier}-logging"
+
+            # Logging config will be set out of this resource
+            # the following `ignore_change` allows to avoid conflicts
+            values.setdefault("lifecycle", {}).setdefault("ignore_changes", []).append(
+                "logging"
+            )
+
+            logging_values = {
+                "bucket": "${aws_s3_bucket." + identifier + ".id}",
+                "target_bucket": "${aws_s3_bucket." + target_bucket_name + ".id}",
+                "target_prefix": s3_bucket_logging.get("target_prefix", ""),
+            }
+            logging_tf_resource = aws_s3_bucket_logging(
+                logging_identifier, **logging_values
+            )
+            tf_resources.append(logging_tf_resource)
         deps = []
         replication_configs = common_values.get("replication_configurations")
         if replication_configs:
@@ -1889,9 +2268,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 # Terraform resource reference:
                 # https://www.terraform.io/docs/providers/aws/r/iam_policy_attachment.html
                 rc_values.clear()
-                rc_values["depends_on"] = self.get_dependencies(
-                    [role_resource, policy_resource]
-                )
+                rc_values["depends_on"] = self.get_dependencies([
+                    role_resource,
+                    policy_resource,
+                ])
                 rc_values["role"] = "${aws_iam_role." + id + ".name}"
                 rc_values["policy_arn"] = "${aws_iam_policy." + id + ".arn}"
                 tf_resource = aws_iam_role_policy_attachment(id, **rc_values)
@@ -1923,14 +2303,14 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             values["provider"] = "aws." + region
         bucket_tf_resource = aws_s3_bucket(identifier, **values)
         tf_resources.append(bucket_tf_resource)
-        output_name_0_13 = output_prefix + "__bucket"
+        output_name = output_prefix + "__bucket"
         output_value = bucket_tf_resource.bucket
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
-        endpoint = "s3.{}.amazonaws.com".format(region)
-        output_name_0_13 = output_prefix + "__endpoint"
-        tf_resources.append(Output(output_name_0_13, value=endpoint))
+        tf_resources.append(Output(output_name, value=output_value))
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
+        endpoint = f"s3.{region}.amazonaws.com"
+        output_name = output_prefix + "__endpoint"
+        tf_resources.append(Output(output_name, value=endpoint))
 
         sqs_identifier = common_values.get("sqs_identifier", None)
         if sqs_identifier is not None:
@@ -2043,6 +2423,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "policy": bucket_policy,
                 "depends_on": self.get_dependencies([bucket_tf_resource]),
             }
+            if self._multiregion_account(account):
+                values["provider"] = "aws." + region
             bucket_policy_tf_resource = aws_s3_bucket_policy(identifier, **values)
             tf_resources.append(bucket_policy_tf_resource)
 
@@ -2065,10 +2447,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # iam user policy for bucket
         values = {}
-        values["user"] = identifier
         values["name"] = identifier
 
-        action = ["s3:*Object"]
+        action = ["s3:*Object*"]
         if common_values.get("acl", "private") == "public-read":
             action.append("s3:PutObjectAcl")
         allow_object_tagging = common_values.get("allow_object_tagging", False)
@@ -2095,12 +2476,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         values["policy"] = json.dumps(policy, sort_keys=True)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
-        # This is temporary, we are going to remove this after the
-        # aws_iam_user_policy_attachment is deployed
-        tf_aws_iam_user_policy = aws_iam_user_policy(identifier, **values)
-        tf_resources.append(tf_aws_iam_user_policy)
-
-        values.pop("user")
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
         tf_resources.append(tf_aws_iam_policy)
 
@@ -2184,23 +2559,23 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         # we want the outputs to be formed into an OpenShift Secret
         # with the following fields
         # db.endpoint
-        output_name_0_13 = output_prefix + "__db_endpoint"
+        output_name = output_prefix + "__db_endpoint"
         # https://docs.aws.amazon.com/AmazonElastiCache/
         # latest/red-ug/Endpoints.html
         if pg_cluster_enabled:
             output_value = "${" + tf_resource.configuration_endpoint_address + "}"
         else:
             output_value = "${" + tf_resource.primary_endpoint_address + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # db.port
-        output_name_0_13 = output_prefix + "__db_port"
+        output_name = output_prefix + "__db_port"
         output_value = "${" + str(tf_resource.port) + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # db.auth_token
         if values.get("transit_encryption_enabled", False):
-            output_name_0_13 = output_prefix + "__db_auth_token"
+            output_name = output_prefix + "__db_auth_token"
             output_value = values["auth_token"]
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
 
         self.add_resources(account, tf_resources)
 
@@ -2245,19 +2620,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 for k, v in data.items():
                     to_replace = "${" + k + "}"
                     user_policy = user_policy.replace(to_replace, v)
-                    output_name_0_13 = output_prefix + "__{}".format(k)
-                    tf_resources.append(Output(output_name_0_13, value=v))
-
-            # This is temporary, we are going to remove this after the
-            # aws_iam_user_policy_attachment is deployed
-            tf_aws_iam_user_policy = aws_iam_user_policy(
-                identifier,
-                name=identifier,
-                user=identifier,
-                policy=user_policy,
-                depends_on=self.get_dependencies([user_tf_resource]),
-            )
-            tf_resources.append(tf_aws_iam_user_policy)
+                    output_name = output_prefix + f"__{k}"
+                    tf_resources.append(Output(output_name, value=v))
 
             tf_aws_iam_policy = aws_iam_policy(
                 identifier,
@@ -2286,15 +2650,15 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             # OCM AWS infrastructure access.
             assume_role = aws_infrastructure_access.get("assume_role")
             if assume_role:
-                output_name_0_13 = output_prefix + "__role_arn"
-                tf_resources.append(Output(output_name_0_13, value=assume_role))
+                output_name = output_prefix + "__role_arn"
+                tf_resources.append(Output(output_name, value=assume_role))
             elif ocm_map:
                 cluster = aws_infrastructure_access["cluster"]["name"]
                 ocm = ocm_map.get(cluster)
                 role_grants = ocm.get_aws_infrastructure_access_role_grants(cluster)
                 for user_arn, _, state, switch_role_link in role_grants:
                     # find correct user by identifier
-                    user_id = awsh.get_user_id_from_arn(user_arn)
+                    user_id = awsh.get_id_from_arn(user_arn)
                     # output will only be added once
                     # terraform-resources created the user
                     # and ocm-aws-infrastructure-access granted it the role
@@ -2302,10 +2666,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         switch_role_arn = awsh.get_role_arn_from_role_link(
                             switch_role_link
                         )
-                        output_name_0_13 = output_prefix + "__role_arn"
-                        tf_resources.append(
-                            Output(output_name_0_13, value=switch_role_arn)
-                        )
+                        output_name = output_prefix + "__role_arn"
+                        tf_resources.append(Output(output_name, value=switch_role_arn))
             else:
                 raise KeyError(
                     f"[{account}/{identifier}] "
@@ -2348,9 +2710,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             )
         )
 
-        output_name_0_13 = output_prefix + "__secrets_prefix"
+        output_name = output_prefix + "__secrets_prefix"
         output_value = secrets_prefix
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -2365,12 +2727,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         assume_role = common_values["assume_role"]
         assume_role = {k: v for k, v in assume_role.items() if v is not None}
+        assume_action = common_values.get("assume_action") or "AssumeRole"
         # assume role policy
         assume_role_policy = {
             "Version": "2012-10-17",
             "Statement": [
                 {
-                    "Action": "sts:AssumeRole",
+                    "Action": f"sts:{assume_action}",
                     "Effect": "Allow",
                     "Principal": assume_role,
                 }
@@ -2391,16 +2754,96 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         if inline_policy:
             values["inline_policy"] = {"name": identifier, "policy": inline_policy}
 
+        if lifecycle := self.get_resource_lifecycle(common_values):
+            values["lifecycle"] = lifecycle
+
         role_tf_resource = aws_iam_role(identifier, **values)
         tf_resources.append(role_tf_resource)
 
+        if role_policy := common_values.get("role_policy"):
+            tf_aws_iam_policy = aws_iam_policy(
+                identifier,
+                name=identifier,
+                policy=role_policy,
+                depends_on=self.get_dependencies([role_tf_resource]),
+                tags=common_values["tags"],
+            )
+            tf_resources.append(tf_aws_iam_policy)
+
+            tf_aws_iam_policy_attachment = aws_iam_role_policy_attachment(
+                identifier,
+                role=role_tf_resource.name,
+                policy_arn=f"${{{tf_aws_iam_policy.arn}}}",
+                depends_on=self.get_dependencies([role_tf_resource, tf_aws_iam_policy]),
+            )
+            tf_resources.append(tf_aws_iam_policy_attachment)
+
+        for policy in common_values.get("policies") or []:
+            tf_iam_role_policy_attachment = aws_iam_role_policy_attachment(
+                identifier + "-" + policy,
+                role=role_tf_resource.name,
+                policy_arn=f"arn:{self._get_partition(account)}:iam::aws:policy/{policy}",
+                depends_on=self.get_dependencies([role_tf_resource]),
+            )
+            tf_resources.append(tf_iam_role_policy_attachment)
+
         # output role arn
-        output_name_0_13 = output_prefix + "__role_arn"
+        output_name = output_prefix + "__role_arn"
         output_value = "${" + role_tf_resource.arn + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
+    def populate_iam_policy(self, account: str, name: str, policy: dict[str, Any]):
+        tf_aws_iam_policy = aws_iam_policy(
+            f"{account}-{name}", name=name, policy=json.dumps(policy)
+        )
+        self.add_resource(account, tf_aws_iam_policy)
+
+    def populate_saml_iam_role(
+        self,
+        account: str,
+        name: str,
+        saml_provider_name: str,
+        aws_managed_policies: list[str],
+        customer_managed_policies: list[str] | None = None,
+        max_session_duration_hours: int = 1,
+    ) -> None:
+        """Manage the an IAM role needed for SAML authentication."""
+        managed_policy_arns = [
+            f"arn:{self._get_partition(account)}:iam::aws:policy/{policy}"
+            for policy in aws_managed_policies
+        ]
+        managed_policy_arns += [
+            f"arn:{self._get_partition(account)}:iam::{self.uids[account]}:policy/{policy}"
+            for policy in customer_managed_policies or []
+        ]
+        assume_role_policy = {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Federated": f"arn:{self._get_partition(account)}:iam::{self.uids[account]}:saml-provider/{saml_provider_name}"
+                    },
+                    "Action": "sts:AssumeRoleWithSAML",
+                    "Condition": {
+                        "StringEquals": {
+                            "SAML:aud": "https://signin.aws.amazon.com/saml"
+                        }
+                    },
+                }
+            ],
+        }
+        role_tf_resource = aws_iam_role(
+            f"{account}-{name}",
+            name=name,
+            assume_role_policy=json.dumps(assume_role_policy),
+            managed_policy_arns=managed_policy_arns,
+            max_session_duration=max_session_duration_hours * 3600,
+        )
+        self.add_resource(account, role_tf_resource)
+
     def populate_tf_resource_sqs(self, spec):
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -2414,9 +2857,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         specs = common_values.get("specs")
         all_queues_per_spec = []
         kms_keys = set()
-        for spec in specs:
-            defaults = self.get_values(spec["defaults"])
-            queues = spec.pop("queues", [])
+        for _spec in specs:
+            defaults = self.get_values(_spec["defaults"])
+            queues = _spec.pop("queues", [])
             all_queues = []
             for queue_kv in queues:
                 queue_key = queue_kv["key"]
@@ -2473,13 +2916,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     kms_keys.add(values["kms_master_key_id"])
                 queue_tf_resource = aws_sqs_queue(queue, **values)
                 tf_resources.append(queue_tf_resource)
-                output_name_0_13 = output_prefix + "__aws_region"
-                tf_resources.append(Output(output_name_0_13, value=region))
-                output_name_0_13 = "{}__{}".format(output_prefix, queue_key)
-                output_value = "https://sqs.{}.amazonaws.com/{}/{}".format(
-                    region, uid, queue_name
-                )
-                tf_resources.append(Output(output_name_0_13, value=output_value))
+                output_name = output_prefix + "__aws_region"
+                tf_resources.append(Output(output_name, value=region))
+                output_name = f"{output_prefix}__{queue_key}"
+                output_value = f"https://sqs.{region}.amazonaws.com/{uid}/{queue_name}"
+                tf_resources.append(Output(output_name, value=output_value))
             all_queues_per_spec.append(all_queues)
 
         # iam resources
@@ -2499,10 +2940,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         )
 
         # iam policy for queue
-        policy_index = 0
-        for all_queues in all_queues_per_spec:
+        for policy_index, all_queues in enumerate(all_queues_per_spec):
             policy_identifier = f"{identifier}-{policy_index}"
-            policy_index += 1
             if len(all_queues_per_spec) == 1:
                 policy_identifier = identifier
             values = {}
@@ -2536,9 +2975,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             values = {}
             values["user"] = identifier
             values["policy_arn"] = "${" + policy_tf_resource.arn + "}"
-            values["depends_on"] = self.get_dependencies(
-                [user_tf_resource, policy_tf_resource]
-            )
+            values["depends_on"] = self.get_dependencies([
+                user_tf_resource,
+                policy_tf_resource,
+            ])
             tf_resource = aws_iam_user_policy_attachment(policy_identifier, **values)
             tf_resources.append(tf_resource)
 
@@ -2554,10 +2994,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         values = {}
         fifo_topic = common_values.get("fifo_topic", False)
-        if fifo_topic:
-            topic_name = identifier + (".fifo")
-        else:
-            topic_name = identifier
+        topic_name = identifier + ".fifo" if fifo_topic else identifier
 
         values["name"] = topic_name
         values["policy"] = policy
@@ -2568,9 +3005,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resource = aws_sns_topic(identifier, **values)
         tf_resources.append(tf_resource)
 
-        if "subscriptions" in common_values.keys():
+        if "subscriptions" in common_values:
             subscriptions = common_values.get("subscriptions")
-            for sub in subscriptions:
+            for index, sub in enumerate(subscriptions):
                 sub_values = {}
                 sub_values["topic_arn"] = "${aws_sns_topic" + "." + identifier + ".arn}"
                 protocol = sub["protocol"]
@@ -2580,17 +3017,21 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     raise ValueError(msg)
                 sub_values["protocol"] = protocol
                 sub_values["endpoint"] = endpoint
-                sub_identifier = f"{identifier}_{protocol}_aws_sns_topic_subscription"
+                # append suffix only in case there are multiple subscriptions
+                suffix = f"_{index + 1}" if len(subscriptions) > 1 else ""
+                sub_identifier = (
+                    f"{identifier}_{protocol}_aws_sns_topic_subscription{suffix}"
+                )
                 sub_tf_resource = aws_sns_topic_subscription(
                     sub_identifier, **sub_values
                 )
                 tf_resources.append(sub_tf_resource)
 
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
-        output_name_0_13 = output_prefix + "__endpoint"
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
+        output_name = output_prefix + "__endpoint"
         output_value = f"https://sns.{region}.amazonaws.com"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         self.add_resources(account, tf_resources)
 
     def populate_tf_resource_dynamodb(self, spec):
@@ -2605,10 +3046,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         region = common_values.get("region") or self.default_regions.get(account)
         specs = common_values.get("specs")
         all_tables = []
-        for spec in specs:
-            defaults = self.get_values(spec["defaults"])
+        for _spec in specs:
+            defaults = self.get_values(_spec["defaults"])
             attributes = defaults.pop("attributes")
-            tables = spec["tables"]
+            tables = _spec["tables"]
             for table_kv in tables:
                 table_key = table_kv["key"]
                 table = table_kv["value"]
@@ -2626,14 +3067,14 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     values["provider"] = "aws." + region
                 table_tf_resource = aws_dynamodb_table(table, **values)
                 tf_resources.append(table_tf_resource)
-                output_name_0_13 = "{}__{}".format(output_prefix, table_key)
-                tf_resources.append(Output(output_name_0_13, value=table))
+                output_name = f"{output_prefix}__{table_key}"
+                tf_resources.append(Output(output_name, value=table))
 
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
-        output_name_0_13 = output_prefix + "__endpoint"
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
+        output_name = output_prefix + "__endpoint"
         output_value = f"https://dynamodb.{region}.amazonaws.com"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         # iam resources
         # Terraform resource reference:
@@ -2653,7 +3094,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # iam user policy for queue
         values = {}
-        values["user"] = identifier
         values["name"] = identifier
         policy = {
             "Version": "2012-10-17",
@@ -2662,8 +3102,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     "Effect": "Allow",
                     "Action": ["dynamodb:*"],
                     "Resource": [
-                        "arn:aws:dynamodb:{}:{}:table/{}".format(region, uid, t)
-                        for t in all_tables
+                        f"arn:aws:dynamodb:{region}:{uid}:table/{t}" for t in all_tables
                     ],
                 }
             ],
@@ -2671,12 +3110,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         values["policy"] = json.dumps(policy, sort_keys=True)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
-        # This is temporary, we are going to remove this after the
-        # aws_iam_user_policy_attachment is deployed
-        tf_aws_iam_user_policy = aws_iam_user_policy(identifier, **values)
-        tf_resources.append(tf_aws_iam_user_policy)
-
-        values.pop("user")
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
         tf_resources.append(tf_aws_iam_policy)
 
@@ -2719,13 +3152,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             values["repository_name"] = values.pop("name")
             ecr_tf_resource = aws_ecrpublic_repository(identifier, **values)
         tf_resources.append(ecr_tf_resource)
-        output_name_0_13 = output_prefix + "__url"
+        output_name = output_prefix + "__url"
         output_value = "${" + ecr_tf_resource.repository_url + "}"
         if public:
             output_value = "${" + ecr_tf_resource.repository_uri + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
+        tf_resources.append(Output(output_name, value=output_value))
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
 
         # iam resources
         # Terraform resource reference:
@@ -2746,7 +3179,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # iam user policy for bucket
         values = {}
-        values["user"] = identifier
         values["name"] = identifier
         policy = {
             "Version": "2012-10-17",
@@ -2787,12 +3219,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         values["policy"] = json.dumps(policy, sort_keys=True)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
-        # This is temporary, we are going to remove this after the
-        # aws_iam_user_policy_attachment is deployed
-        tf_aws_iam_user_policy = aws_iam_user_policy(identifier, **values)
-        tf_resources.append(tf_aws_iam_user_policy)
-
-        values.pop("user")
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
         tf_resources.append(tf_aws_iam_policy)
 
@@ -2850,49 +3276,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         bucket_policy_tf_resource = aws_s3_bucket_policy(identifier, **values)
         tf_resources.append(bucket_policy_tf_resource)
 
-        # cloud front distribution
         values = common_values.get("distribution_config", {})
-        values["tags"] = common_values["tags"]
-        values.setdefault("default_cache_behavior", {}).setdefault(
-            "target_origin_id", "default"
-        )
-        origin = {
-            "domain_name": "${" + bucket_tf_resource.bucket_domain_name + "}",
-            "origin_id": values["default_cache_behavior"]["target_origin_id"],
-            "s3_origin_config": {
-                "origin_access_identity": "origin-access-identity/cloudfront/"
-                + "${"
-                + cf_oai_tf_resource.id
-                + "}"
-            },
-        }
-        values["origin"] = [origin]
-        cf_distribution_tf_resource = aws_cloudfront_distribution(identifier, **values)
-        tf_resources.append(cf_distribution_tf_resource)
-
-        # outputs
-        # cloud_front_origin_access_identity_id
-        output_name_0_13 = output_prefix + "__cloud_front_origin_access_identity_id"
-        output_value = "${" + cf_oai_tf_resource.id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        # s3_canonical_user_id
-        output_name_0_13 = output_prefix + "__s3_canonical_user_id"
-        output_value = "${" + cf_oai_tf_resource.s3_canonical_user_id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        # distribution_domain
-        output_name_0_13 = output_prefix + "__distribution_domain"
-        output_value = "${" + cf_distribution_tf_resource.domain_name + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        # origin_access_identity
-        output_name_0_13 = output_prefix + "__origin_access_identity"
-        output_value = (
-            "origin-access-identity/cloudfront/" + "${" + cf_oai_tf_resource.id + "}"
-        )
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-
         # aws_s3_bucket_acl
-        values = common_values.get("distribution_config", {})
-        if "logging_config" in values.keys():
+        if "logging_config" in values:
             # we could set this at a global level with a standard name like "cloudfront"
             # but we need all aws accounts upgraded to aws provider >3.60 first
             tf_resources.append(
@@ -2900,7 +3286,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             )
 
             logging_config_bucket = values["logging_config"]
-            values = {}
+            acl_values = {}
             access_control_policy = {
                 "owner": {
                     "id": "${data.aws_canonical_user_id.current.id}",
@@ -2923,12 +3309,61 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     },
                 ],
             }
-            values["access_control_policy"] = access_control_policy
-            values["bucket"] = logging_config_bucket.get("bucket").split(".")[0]
+            external_account_id = logging_config_bucket.pop("external_account_id", None)
+            if external_account_id:
+                external_account_policy = {
+                    "grantee": {
+                        "id": external_account_id,
+                        "type": "CanonicalUser",
+                    },
+                    "permission": "FULL_CONTROL",
+                }
+                access_control_policy["grant"].append(external_account_policy)
+            acl_values["access_control_policy"] = access_control_policy
+            acl_values["bucket"] = logging_config_bucket.get("bucket").split(".")[0]
 
-            aws_s3_bucket_acl_resource = aws_s3_bucket_acl(identifier, **values)
+            aws_s3_bucket_acl_resource = aws_s3_bucket_acl(identifier, **acl_values)
             tf_resources.append(aws_s3_bucket_acl_resource)
 
+        # cloud front distribution
+        values["tags"] = common_values["tags"]
+        values.setdefault("default_cache_behavior", {}).setdefault(
+            "target_origin_id", "default"
+        )
+        origin = {
+            "domain_name": "${" + bucket_tf_resource.bucket_domain_name + "}",
+            "origin_id": values["default_cache_behavior"]["target_origin_id"],
+            "s3_origin_config": {
+                "origin_access_identity": "origin-access-identity/cloudfront/"
+                + "${"
+                + cf_oai_tf_resource.id
+                + "}"
+            },
+        }
+        values["origin"] = [origin]
+        cf_distribution_tf_resource = aws_cloudfront_distribution(identifier, **values)
+        tf_resources.append(cf_distribution_tf_resource)
+
+        # outputs
+        # cloud_front_origin_access_identity_id
+        output_name = output_prefix + "__cloud_front_origin_access_identity_id"
+        output_value = "${" + cf_oai_tf_resource.id + "}"
+        tf_resources.append(Output(output_name, value=output_value))
+        # s3_canonical_user_id
+        output_name = output_prefix + "__s3_canonical_user_id"
+        output_value = "${" + cf_oai_tf_resource.s3_canonical_user_id + "}"
+        tf_resources.append(Output(output_name, value=output_value))
+        # distribution_domain
+        output_name = output_prefix + "__distribution_domain"
+        output_value = "${" + cf_distribution_tf_resource.domain_name + "}"
+        tf_resources.append(Output(output_name, value=output_value))
+        # origin_access_identity
+        output_name = output_prefix + "__origin_access_identity"
+        output_value = (
+            "origin-access-identity/cloudfront/" + "${" + cf_oai_tf_resource.id + "}"
+        )
+        tf_resources.append(Output(output_name, value=output_value))
+
         self.add_resources(account, tf_resources)
 
     def populate_tf_resource_s3_sqs(self, spec):
@@ -3076,13 +3511,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources.append(access_key_tf_resource)
         # outputs
         # sqs_aws_access_key_id
-        output_name_0_13 = output_prefix + "__sqs_aws_access_key_id"
+        output_name = output_prefix + "__sqs_aws_access_key_id"
         output_value = "${" + access_key_tf_resource.id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value, sensitive=True))
         # sqs_aws_secret_access_key
-        output_name_0_13 = output_prefix + "__sqs_aws_secret_access_key"
+        output_name = output_prefix + "__sqs_aws_secret_access_key"
         output_value = "${" + access_key_tf_resource.secret + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value, sensitive=True))
 
         # iam policy for queue
         values = {}
@@ -3116,20 +3551,19 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         values = {}
         values["user"] = sqs_identifier
         values["policy_arn"] = "${" + policy_tf_resource.arn + "}"
-        values["depends_on"] = self.get_dependencies(
-            [user_tf_resource, policy_tf_resource]
-        )
+        values["depends_on"] = self.get_dependencies([
+            user_tf_resource,
+            policy_tf_resource,
+        ])
         user_policy_attachment_tf_resource = aws_iam_user_policy_attachment(
             sqs_identifier, **values
         )
         tf_resources.append(user_policy_attachment_tf_resource)
 
         # outputs
-        output_name_0_13 = "{}__{}".format(output_prefix, sqs_identifier)
-        output_value = "https://sqs.{}.amazonaws.com/{}/{}".format(
-            region, uid, sqs_identifier
-        )
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        output_name = f"{output_prefix}__{sqs_identifier}"
+        output_value = f"https://sqs.{region}.amazonaws.com/{uid}/{sqs_identifier}"
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -3294,11 +3728,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             )
             tf_resources.append(subscription_tf_resource)
 
-        output_name_0_13 = output_prefix + "__log_group_name"
+        output_name = output_prefix + "__log_group_name"
         output_value = log_group_tf_resource.name
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
+        tf_resources.append(Output(output_name, value=output_value))
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
 
         # iam resources
         # Terraform resource reference:
@@ -3340,18 +3774,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             ],
         }
         values = {
-            "user": identifier,
             "name": identifier,
             "policy": json.dumps(policy, sort_keys=True),
             "depends_on": self.get_dependencies([user_tf_resource]),
         }
 
-        # This is temporary, we are going to remove this after the
-        # aws_iam_user_policy_attachment is deployed
-        tf_aws_iam_user_policy = aws_iam_user_policy(identifier, **values)
-        tf_resources.append(tf_aws_iam_user_policy)
-
-        values.pop("user")
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
         tf_resources.append(tf_aws_iam_policy)
 
@@ -3395,9 +3822,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources.append(tf_resource)
 
         # key_id
-        output_name_0_13 = output_prefix + "__key_id"
+        output_name = output_prefix + "__key_id"
         output_value = "${" + tf_resource.key_id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         alias_values = {}
         alias_values["name"] = "alias/" + identifier
@@ -3510,11 +3937,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             attachment_values = {
                 "role": role_tf_resource.name,
                 "policy_arn": "${" + policy_tf_resource.arn + "}",
-                "depends_on": self.get_dependencies(
-                    [
-                        role_tf_resource,
-                    ]
-                ),
+                "depends_on": self.get_dependencies([
+                    role_tf_resource,
+                ]),
             }
             attachment_tf_resource = aws_iam_role_policy_attachment(
                 policy_identifier, **attachment_values
@@ -3598,9 +4023,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     "starting_position_timestamp", None
                 )
                 if not starting_position_timestamp:
-                    source_vaules[
-                        "starting_position_timestamp"
-                    ] = starting_position_timestamp
+                    source_vaules["starting_position_timestamp"] = (
+                        starting_position_timestamp
+                    )
 
             batch_size = common_values.get("batch_size", 100)
             source_vaules["batch_size"] = batch_size
@@ -3617,11 +4042,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # outputs
         # stream_name
-        output_name_0_13 = output_prefix + "__stream_name"
-        tf_resources.append(Output(output_name_0_13, value=identifier))
+        output_name = output_prefix + "__stream_name"
+        tf_resources.append(Output(output_name, value=identifier))
         # aws_region
-        output_name_0_13 = output_prefix + "__aws_region"
-        tf_resources.append(Output(output_name_0_13, value=region))
+        output_name = output_prefix + "__aws_region"
+        tf_resources.append(Output(output_name, value=region))
 
         # iam resources
         policy = {
@@ -3713,17 +4138,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # iam user policy
         values = {}
-        values["user"] = identifier
         values["name"] = identifier
         values["policy"] = json.dumps(policy, sort_keys=True)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
-        # This is temporary, we are going to remove this after the
-        # aws_iam_user_policy_attachment is deployed
-        tf_aws_iam_user_policy = aws_iam_user_policy(identifier, **values)
-        tf_resources.append(tf_aws_iam_user_policy)
-
-        values.pop("user")
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
         tf_resources.append(tf_aws_iam_policy)
 
@@ -3746,13 +4164,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources.append(tf_resource)
         # outputs
         # aws_access_key_id
-        output_name_0_13 = output_prefix + "__aws_access_key_id"
+        output_name = output_prefix + "__aws_access_key_id"
         output_value = "${" + tf_resource.id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value, sensitive=True))
         # aws_secret_access_key
-        output_name_0_13 = output_prefix + "__aws_secret_access_key"
+        output_name = output_prefix + "__aws_secret_access_key"
         output_value = "${" + tf_resource.secret + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value, sensitive=True))
 
         return tf_resources
 
@@ -3763,17 +4181,30 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
     def add_resource(self, account, tf_resource):
         if account not in self.locks:
             logging.debug(
-                "integration {} is disabled for account {}. "
-                "can not add resource".format(self.integration, account)
+                f"integration {self.integration} is disabled for account {account}. "
+                "can not add resource"
             )
             return
         with self.locks[account]:
             self.tss[account].add(tf_resource)
 
+    def add_moved(self, account: str, moved: Moved):
+        if account not in self.locks:
+            logging.debug(
+                f"integration {self.integration} is disabled for account {account}. "
+                "can not add resource"
+            )
+            return
+        with self.locks[account]:
+            self.tss[account].setdefault("moved", []).append({
+                "from": moved.fro,
+                "to": moved.to,
+            })
+
     def dump(
         self,
-        print_to_file: Optional[str] = None,
-        existing_dirs: Optional[dict[str, str]] = None,
+        print_to_file: str | None = None,
+        existing_dirs: dict[str, str] | None = None,
     ) -> dict[str, str]:
         """
         Dump the Terraform configurations (in JSON format) to the working directories.
@@ -3796,21 +4227,36 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 os.remove(print_to_file)
 
         for name, ts in self.tss.items():
+            content = str(ts)
             if print_to_file:
-                with open(print_to_file, "a") as f:
+                with open(print_to_file, "a", encoding="locale") as f:
                     f.write(f"##### {name} #####\n")
-                    f.write(str(ts))
+                    f.write(content)
                     f.write("\n")
             if existing_dirs is None:
                 wd = tempfile.mkdtemp(prefix=TMP_DIR_PREFIX)
             else:
                 wd = working_dirs[name]
-            with open(wd + "/config.tf.json", "w") as f:
-                f.write(str(ts))
+            with open(wd + "/config.tf.json", "w", encoding="locale") as f:
+                f.write(content)
             working_dirs[name] = wd
 
         return working_dirs
 
+    def terraform_configurations(self) -> dict[str, str]:
+        """
+        Return the Terraform configurations (in JSON format) for each AWS account.
+        Terraform config content keys are sorted for consistent hash check.
+        The result is not used for final file dump.
+        Doc: https://python-terrascript.readthedocs.io/en/stable/quickstart.html
+
+        :return: key is AWS account name and value is terraform configuration
+        """
+        return {
+            name: json.dumps(ts, indent=2, sort_keys=True)
+            for name, ts in self.tss.items()
+        }
+
     def init_values(self, spec: ExternalResourceSpec, init_tags: bool = True) -> dict:
         """
         Initialize the values of the terraform resource and merge the defaults and
@@ -3834,6 +4280,33 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             if val is not None:
                 values[key] = val
 
+        if self.versions and not self.versions.get(
+            spec.provisioner_name, ""
+        ).startswith("3"):
+            if spec.provider == "rds":
+                if db_name := values.pop("name", None):
+                    values["db_name"] = db_name
+                if values.get("replica_source"):
+                    values.pop("db_name", None)
+            elif spec.provider == "elasticache":
+                if description := values.pop("replication_group_description", None):
+                    values["description"] = description
+                if num_cache_clusters := values.pop("number_cache_clusters", None):
+                    values["num_cache_clusters"] = num_cache_clusters
+                if cluster_mode := values.pop("cluster_mode", {}):
+                    for k, v in cluster_mode.items():
+                        values[k] = v
+                values.pop("availability_zones", None)
+            elif spec.provider == "msk":
+                if ebs_volume_size := values.get("broker_node_group_info", {}).pop(
+                    "ebs_volume_size", None
+                ):
+                    values["broker_node_group_info"].setdefault(
+                        "storage_info", {}
+                    ).setdefault("ebs_storage_info", {})[
+                        "volume_size"
+                    ] = ebs_volume_size
+
         return values
 
     @staticmethod
@@ -3863,39 +4336,39 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources: list[Resource],
         spec: ExternalResourceSpec,
     ):
-        output_format_0_13 = "{}__{}_{}"
+        output_format = "{}__{}_{}"
         # cluster
-        output_name_0_13 = output_format_0_13.format(
+        output_name = output_format.format(
             spec.output_prefix, self.integration_prefix, "cluster"
         )
         output_value = spec.cluster_name
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # namespace
-        output_name_0_13 = output_format_0_13.format(
+        output_name = output_format.format(
             spec.output_prefix, self.integration_prefix, "namespace"
         )
         output_value = spec.namespace_name
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # resource
-        output_name_0_13 = output_format_0_13.format(
+        output_name = output_format.format(
             spec.output_prefix, self.integration_prefix, "resource"
         )
         output_value = "Secret"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # output_resource_name
-        output_name_0_13 = output_format_0_13.format(
+        output_name = output_format.format(
             spec.output_prefix, self.integration_prefix, "output_resource_name"
         )
         output_value = spec.output_resource_name
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # annotations
         if spec.annotations():
-            output_name_0_13 = output_format_0_13.format(
+            output_name = output_format.format(
                 spec.output_prefix, self.integration_prefix, "annotations"
             )
             anno_json = json.dumps(spec.annotations()).encode("utf-8")
             output_value = base64.b64encode(anno_json).decode()
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value))
 
     def prefetch_resources(self, schema) -> dict[str, dict[str, str]]:
         gqlapi = gql.get_api()
@@ -3909,7 +4382,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         try:
             raw_values = gqlapi.get_resource(path)
         except gql.GqlGetResourceError as e:
-            raise FetchResourceError(str(e))
+            raise FetchResourceError(str(e)) from e
         return raw_values
 
     def get_values(self, path: str) -> dict[str, Any]:
@@ -3919,7 +4392,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             values.pop("$schema", None)
         except anymarkup.AnyMarkupError:
             e_msg = "Could not parse data. Skipping resource: {}"
-            raise FetchResourceError(e_msg.format(path))
+            raise FetchResourceError(e_msg.format(path)) from None
         return values
 
     @staticmethod
@@ -3990,7 +4463,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
     def _get_elasticsearch_account_wide_resource_policy(
         self, account: str
-    ) -> Optional[aws_cloudwatch_log_resource_policy]:
+    ) -> aws_cloudwatch_log_resource_policy | None:
         """
         https://docs.aws.amazon.com/opensearch-service/latest/developerguide/createdomain-configure-slow-logs.html
         CloudWatch Logs supports 10 resource policies per Region.
@@ -4065,13 +4538,11 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         for t in ElasticSearchLogGroupType:
             log_type = t.value
             if log_type not in publish_log_types:
-                publishing_options.append(
-                    {
-                        "log_type": log_type,
-                        "enabled": False,
-                        "cloudwatch_log_group_arn": "",
-                    }
-                )
+                publishing_options.append({
+                    "log_type": log_type,
+                    "enabled": False,
+                    "cloudwatch_log_group_arn": "",
+                })
                 continue
 
             log_type_identifier = TerrascriptClient.elasticsearch_log_group_identifier(
@@ -4093,25 +4564,23 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             arn = f"${{{log_group_tf_resource.arn}}}"
 
             # add arn to output
-            output_name_0_13 = (
+            output_name = (
                 f"{output_prefix}__cloudwatch_log_group_" f"{log_type.lower()}_arn"
             )
             output_value = arn
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value))
 
             # add name to output
-            output_name_0_13 = (
+            output_name = (
                 f"{output_prefix}__cloudwatch_log_group_" f"{log_type.lower()}_name"
             )
             output_value = log_type_identifier
-            tf_resources.append(Output(output_name_0_13, value=output_value))
-            publishing_options.append(
-                {
-                    "log_type": log_type,
-                    "enabled": True,
-                    "cloudwatch_log_group_arn": arn,
-                }
-            )
+            tf_resources.append(Output(output_name, value=output_value))
+            publishing_options.append({
+                "log_type": log_type,
+                "enabled": True,
+                "cloudwatch_log_group_arn": arn,
+            })
 
         return tf_resources, publishing_options
 
@@ -4281,6 +4750,24 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             )
             cluster_vaules["warm_count"] = cluster_config.get("warm_count", 2)
 
+        cold_storage_options = cluster_config.get("cold_storage_options", {})
+        if cold_storage_options.get("enabled", False):
+            if not dedicated_master_enabled:
+                raise ElasticSearchResourceColdStorageError(
+                    f"[{account}] Cold storage can only be enabled when "
+                    + "dedicated_master_enabled is set to true for resource"
+                    f" {values['identifier']}"
+                )
+            if not warm_enabled:
+                raise ElasticSearchResourceColdStorageError(
+                    f"[{account}] Cold storage can only be enabled when "
+                    + "warm_enabled is set to true for resource"
+                    f" {values['identifier']}"
+                )
+            cluster_vaules["cold_storage_options"] = {
+                "enabled": True,
+            }
+
         es_values["cluster_config"] = cluster_vaules
 
         snapshot_options = values.get("snapshot_options", {})
@@ -4358,19 +4845,19 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         auth_options = values.get("auth", {})
         # TODO: @fishi0x01 make mandatory after migration APPSRE-3409
         if auth_options:
-            es_values[
-                "advanced_security_options"
-            ] = self._build_es_advanced_security_options(auth_options)
+            es_values["advanced_security_options"] = (
+                self._build_es_advanced_security_options(auth_options)
+            )
 
         # TODO: @fishi0x01 remove after migration APPSRE-3409
         # ++++++++ START: REMOVE +++++++++
         else:
             advanced_security_options = values.get("advanced_security_options", {})
             if advanced_security_options:
-                es_values[
-                    "advanced_security_options"
-                ] = self._build_es_advanced_security_options_deprecated(
-                    advanced_security_options
+                es_values["advanced_security_options"] = (
+                    self._build_es_advanced_security_options_deprecated(
+                        advanced_security_options
+                    )
                 )
         # ++++++++ END: REMOVE ++++++++++
 
@@ -4379,33 +4866,33 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # Setup outputs
         # arn
-        output_name_0_13 = output_prefix + "__arn"
+        output_name = output_prefix + "__arn"
         output_value = "${" + es_tf_resource.arn + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # domain_id
-        output_name_0_13 = output_prefix + "__domain_id"
+        output_name = output_prefix + "__domain_id"
         output_value = "${" + es_tf_resource.domain_id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # domain_name
-        output_name_0_13 = output_prefix + "__domain_name"
+        output_name = output_prefix + "__domain_name"
         output_value = es_tf_resource.domain_name
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # endpoint
-        output_name_0_13 = output_prefix + "__endpoint"
+        output_name = output_prefix + "__endpoint"
         output_value = "https://" + "${" + es_tf_resource.endpoint + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # kibana_endpoint
-        output_name_0_13 = output_prefix + "__kibana_endpoint"
+        output_name = output_prefix + "__kibana_endpoint"
         output_value = "https://" + "${" + es_tf_resource.kibana_endpoint + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # vpc_id
-        output_name_0_13 = output_prefix + "__vpc_id"
+        output_name = output_prefix + "__vpc_id"
         output_value = (
             "${aws_elasticsearch_domain." + identifier + ".vpc_options.0.vpc_id}"
         )
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # add master user creds to output and secretsmanager if internal_user_database_enabled
-        security_options = es_values.get("advanced_security_options", None)
+        security_options = es_values.get("advanced_security_options")
         if security_options and security_options.get(
             "internal_user_database_enabled", False
         ):
@@ -4454,20 +4941,20 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             )
             tf_resources.append(iam_policy_resource)
 
-            output_name_0_13 = output_prefix + "__secret_name"
+            output_name = output_prefix + "__secret_name"
             output_value = secret_name
-            tf_resources.append(Output(output_name_0_13, value=output_value))
-            output_name_0_13 = output_prefix + "__secret_policy_arn"
+            tf_resources.append(Output(output_name, value=output_value))
+            output_name = output_prefix + "__secret_policy_arn"
             output_value = "${" + iam_policy_resource.arn + "}"
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value))
             # master_user_name
-            output_name_0_13 = output_prefix + "__master_user_name"
+            output_name = output_prefix + "__master_user_name"
             output_value = master_user["master_user_name"]
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
             # master_user_password
-            output_name_0_13 = output_prefix + "__master_user_password"
+            output_name = output_prefix + "__master_user_password"
             output_value = master_user["master_user_password"]
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
 
         self.add_resources(account, tf_resources)
 
@@ -4545,34 +5032,36 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # outputs
         # arn
-        output_name_0_13 = output_prefix + "__arn"
+        output_name = output_prefix + "__arn"
         output_value = "${" + acm_tf_resource.arn + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # domain name
-        # output_name_0_13 = output_prefix + '__domain_name'
+        # output_name = output_prefix + '__domain_name'
         # output_value = '${' + acm_tf_resource.domain_name + '}'
-        # tf_resources.append(Output(output_name_0_13, value=output_value))
+        # tf_resources.append(Output(output_name, value=output_value))
         # status
-        output_name_0_13 = output_prefix + "__status"
+        output_name = output_prefix + "__status"
         output_value = "${" + acm_tf_resource.status + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # domain_validation_options
-        output_name_0_13 = output_prefix + "__domain_validation_options"
+        output_name = output_prefix + "__domain_validation_options"
         output_value = "${" + acm_tf_resource.domain_validation_options + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         if secret is not None:
             # key
-            output_name_0_13 = output_prefix + "__key"
+            output_name = output_prefix + "__key"
             output_value = key
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value))
             # certificate
-            output_name_0_13 = output_prefix + "__certificate"
+            output_name = output_prefix + "__certificate"
             output_value = certificate
-            tf_resources.append(Output(output_name_0_13, value=output_value))
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
             if caCertificate is not None:
-                output_name_0_13 = output_prefix + "__caCertificate"
+                output_name = output_prefix + "__caCertificate"
                 output_value = caCertificate
-                tf_resources.append(Output(output_name_0_13, value=output_value))
+                tf_resources.append(
+                    Output(output_name, value=output_value, sensitive=True)
+                )
 
         self.add_resources(account, tf_resources)
 
@@ -4609,17 +5098,17 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # outputs
         # etag
-        output_name_0_13 = output_prefix + "_etag"
+        output_name = output_prefix + "_etag"
         output_value = "${" + pk_tf_resource.etag + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # id
-        output_name_0_13 = output_prefix + "__id"
+        output_name = output_prefix + "__id"
         output_value = "${" + pk_tf_resource.id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # key
-        output_name_0_13 = output_prefix + "__key"
+        output_name = output_prefix + "__key"
         output_value = key
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -4629,16 +5118,18 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         account = self.accounts[account_name]
         cluster = namespace_info["cluster"]
         ocm = ocm_map.get(cluster["name"])
-        account[
-            "assume_role"
-        ] = ocm.get_aws_infrastructure_access_terraform_assume_role(
-            cluster["name"],
-            account["uid"],
-            account["terraformUsername"],
+        account["assume_role"] = (
+            ocm.get_aws_infrastructure_access_terraform_assume_role(
+                cluster["name"],
+                account["uid"],
+                account["terraformUsername"],
+            )
         )
         account["assume_region"] = cluster["spec"]["region"]
         service_name = f"{namespace_info['name']}/{openshift_service}"
-        with AWSApi(1, [account], settings=self.settings, init_users=False) as awsapi:
+        with AWSApi(
+            1, [account], secret_reader=self.secret_reader, init_users=False
+        ) as awsapi:
             ips = awsapi.get_alb_network_interface_ips(account, service_name)
         if not ips:
             raise ValueError(
@@ -4652,7 +5143,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
     def _get_alb_rule_condition_value(condition):
         condition_type = condition["type"]
         condition_type_key = SUPPORTED_ALB_LISTENER_RULE_CONDITION_TYPE_MAPPING.get(
-            condition_type, None
+            condition_type
         )
         if condition_type_key is None:
             raise KeyError(f"unknown alb rule condition type {condition_type}")
@@ -4739,7 +5230,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources.append(sg_tf_resource)
 
         # https://www.terraform.io/docs/providers/aws/r/lb.html
-        values = {
+        lb_values = {
             "provider": provider,
             "name": identifier,
             "internal": False,
@@ -4753,13 +5244,33 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         idle_timeout = resource.get("idle_timeout")
         if idle_timeout:
-            values["idle_timeout"] = idle_timeout
+            lb_values["idle_timeout"] = idle_timeout
 
         enable_http2 = resource.get("enable_http2")
         if enable_http2 is False:
-            values["enable_http2"] = False
+            lb_values["enable_http2"] = False
+
+        if resource.get("access_logs"):
+            # https://www.terraform.io/docs/providers/aws/r/lb.html#access_logs
+            bucket_identifier = f"{identifier}-lb-access-logs"
+            lb_access_logs_s3_bucket_values = {
+                "provider": provider,
+                "bucket": bucket_identifier,
+            }
+            lb_access_logs_s3_bucket_tf_resource = aws_s3_bucket(
+                bucket_identifier, **lb_access_logs_s3_bucket_values
+            )
+            tf_resources.append(lb_access_logs_s3_bucket_tf_resource)
 
-        lb_tf_resource = aws_lb(identifier, **values)
+            lb_values["access_logs"] = {
+                "enabled": True,
+                "bucket": f"${{{lb_access_logs_s3_bucket_tf_resource.id}}}",
+            }
+            lb_values["depends_on"].extend(
+                self.get_dependencies([lb_access_logs_s3_bucket_tf_resource])
+            )
+
+        lb_tf_resource = aws_lb(identifier, **lb_values)
         tf_resources.append(lb_tf_resource)
 
         default_target = None
@@ -4768,6 +5279,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             target_name = t["name"]
             t_openshift_service = t.get("openshift_service")
             t_ips = t.get("ips")
+            t_protocol = t.get("protocol") or "HTTPS"
+            t_protocol_version = t.get("protocol_version") or "HTTP1"
+
             if t_openshift_service:
                 target_ips = self._get_alb_target_ips_by_openshift_service(
                     identifier, t_openshift_service, account, namespace_info, ocm_map
@@ -4798,8 +5312,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "provider": provider,
                 "name": f"{target_name}-${{{lbt_random_id.hex}}}",
                 "port": 443,
-                "protocol": "HTTPS",
-                "protocol_version": "HTTP1",
+                "protocol": t_protocol,
+                "protocol_version": t_protocol_version,
                 "target_type": "ip",
                 "vpc_id": vpc_id,
                 "health_check": {
@@ -4832,7 +5346,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     "port": 443,
                     "depends_on": self.get_dependencies([lbt_tf_resource]),
                 }
-                if not ip_address(ip) in ip_network(vpc_cidr_block):
+                if ip_address(ip) not in ip_network(vpc_cidr_block):
                     values["availability_zone"] = "all"
                 ip_slug = ip.replace(".", "_")
                 lbta_identifier = f"{lbt_identifier}-{ip_slug}"
@@ -4864,12 +5378,13 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         # forward
         if not default_target:
             raise KeyError("expected a single default target")
+        ssl_policy = resource.get("ssl_policy") or "ELBSecurityPolicy-TLS13-1-0-2021-06"
         values = {
             "provider": provider,
             "load_balancer_arn": f"${{{lb_tf_resource.arn}}}",
             "port": 443,
             "protocol": "HTTPS",
-            "ssl_policy": "ELBSecurityPolicy-TLS-1-2-2017-01",
+            "ssl_policy": ssl_policy,
             "certificate_arn": resource["certificate_arn"],
             "default_action": {
                 "type": "forward",
@@ -4907,9 +5422,10 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
                     target_resource = valid_targets[target_name]
 
-                    action_values["forward"]["target_group"].append(
-                        {"arn": f"${{{target_resource.arn}}}", "weight": a["weight"]}
-                    )
+                    action_values["forward"]["target_group"].append({
+                        "arn": f"${{{target_resource.arn}}}",
+                        "weight": a["weight"],
+                    })
                     weight_sum += a["weight"]
                 if weight_sum != 100:
                     raise ValueError(
@@ -4926,6 +5442,19 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                         "status_code": fr_data.get("status_code"),
                     },
                 }
+            elif action_type == "redirect":
+                redirect_data = action.get("redirect", {})
+                action_values = {
+                    "type": "redirect",
+                    "redirect": {
+                        "host": redirect_data.get("host"),
+                        "path": redirect_data.get("path"),
+                        "port": redirect_data.get("port"),
+                        "protocol": redirect_data.get("protocol"),
+                        "query": redirect_data.get("query"),
+                        "status_code": redirect_data.get("status_code"),
+                    },
+                }
             else:
                 raise KeyError(f"unknown alb rule action type {action_type}")
 
@@ -4941,20 +5470,20 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "depends_on": self.get_dependencies([forward_lbl_tf_resource]),
             }
 
-            lblr_identifier = f"{identifier}-rule-{rule_num+1:02d}"
+            lblr_identifier = f"{identifier}-rule-{rule_num + 1:02d}"
             lblr_tf_resource = aws_lb_listener_rule(lblr_identifier, **values)
 
             tf_resources.append(lblr_tf_resource)
 
         # outputs
         # dns name
-        output_name_0_13 = output_prefix + "__dns_name"
+        output_name = output_prefix + "__dns_name"
         output_value = f"${{{lb_tf_resource.dns_name}}}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
         # vpc cidr block
-        output_name_0_13 = output_prefix + "__vpc_cidr_block"
+        output_name = output_prefix + "__vpc_cidr_block"
         output_value = vpc_cidr_block
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -4993,12 +5522,12 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         tf_resources.append(aws_version_resource)
 
         # outputs
-        output_name_0_13 = output_prefix + "__arn"
+        output_name = output_prefix + "__arn"
         output_value = "${" + aws_version_resource.arn + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        output_name_0_13 = output_prefix + "__version_id"
+        tf_resources.append(Output(output_name, value=output_value))
+        output_name = output_prefix + "__version_id"
         output_value = "${" + aws_version_resource.version_id + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -5021,14 +5550,14 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         if "gitlab" in url:
             gitlab = self.init_gitlab()
             project = gitlab.get_project(url)
-            commits = project.commits.list(ref_name=ref)
+            commits = project.commits.list(ref_name=ref, per_page=1, page=1)
             return commits[0].id
 
         return ""
 
     def get_asg_image_id(
         self, filters: Iterable[Mapping[str, Any]], account: str, region: str
-    ) -> Optional[str]:
+    ) -> str | None:
         """
         AMI ID comes form AWS Api filter result.
         AMI needs to be shared by integration aws-ami-share.
@@ -5045,7 +5574,9 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         # Get the most recent AMI id
         aws_account = self.accounts[account]
-        with AWSApi(1, [aws_account], settings=self.settings, init_users=False) as aws:
+        with AWSApi(
+            1, [aws_account], secret_reader=self.secret_reader, init_users=False
+        ) as aws:
             return aws.get_image_id(account, region, tags)
 
     def _use_previous_image_id(self, filters: Iterable[Mapping[str, Any]]) -> bool:
@@ -5127,23 +5658,19 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             part = []
             for c in cloudinit_configs:
                 raw = self.get_raw_values(c["content"])
-                content = orb.process_extracurlyjinja2_template(
-                    body=raw["content"], vars=vars, settings=self.settings
+                content = process_extracurlyjinja2_template(
+                    body=raw["content"], vars=vars, secret_reader=self.secret_reader
                 )
                 # https://www.terraform.io/docs/language/expressions/strings.html#escape-sequences
                 content = content.replace("${", "$${")
                 content = content.replace("%{", "%%{")
-                part.append(
-                    {
-                        "filename": c.get("filename"),
-                        "content_type": c.get("content_type"),
-                        "content": content,
-                    }
-                )
+                part.append({
+                    "filename": c.get("filename"),
+                    "content_type": c.get("content_type"),
+                    "content": content,
+                })
             cloudinit_value = {"gzip": True, "base64_encode": True, "part": part}
-            cloudinit_data = data.template_cloudinit_config(
-                identifier, **cloudinit_value
-            )
+            cloudinit_data = cloudinit_config(identifier, **cloudinit_value)
             tf_resources.append(cloudinit_data)
             template_values["user_data"] = "${" + cloudinit_data.rendered + "}"
         template_resource = aws_launch_template(identifier, **template_values)
@@ -5172,25 +5699,36 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 "preferences": common_values.get("instance_refresh_preferences"),
             },
         }
+
+        override = []
         instance_types = common_values.get("instance_types")
         if instance_types:
-            override = [{"instance_type": i} for i in instance_types]
-            asg_value["mixed_instances_policy"]["launch_template"][
-                "override"
-            ] = override
-        asg_value["tags"] = [
+            override += [{"instance_type": i} for i in instance_types]
+        instance_requirements = common_values.get("instance_requirements")
+        if instance_requirements:
+            override += [{"instance_requirements": instance_requirements}]
+        if override:
+            asg_value["mixed_instances_policy"]["launch_template"]["override"] = (
+                override
+            )
+
+        tags = [
             {"key": k, "value": v, "propagate_at_launch": True} for k, v in tags.items()
         ]
+        if self.versions.get(account, "").startswith("3"):
+            asg_value["tags"] = tags
+        else:
+            asg_value["tag"] = tags
         asg_resource = aws_autoscaling_group(identifier, **asg_value)
         tf_resources.append(asg_resource)
 
         # outputs
-        output_name_0_13 = output_prefix + "__template_latest_version"
+        output_name = output_prefix + "__template_latest_version"
         output_value = "${" + template_resource.latest_version + "}"
-        tf_resources.append(Output(output_name_0_13, value=output_value))
-        output_name_0_13 = output_prefix + "__image_id"
+        tf_resources.append(Output(output_name, value=output_value))
+        output_name = output_prefix + "__image_id"
         output_value = image_id
-        tf_resources.append(Output(output_name_0_13, value=output_value))
+        tf_resources.append(Output(output_name, value=output_value))
 
         self.add_resources(account, tf_resources)
 
@@ -5257,7 +5795,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         lambda_managed_policy_arn = (
             "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
         )
-        if region in ("us-gov-west-1", "us-gov-east-1"):
+        if region in {"us-gov-west-1", "us-gov-east-1"}:
             lambda_managed_policy_arn = "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
         vpc_id = common_values.get("vpc_id")
         subnet_ids = common_values.get("subnet_ids")
@@ -5333,8 +5871,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 s3_client.head_bucket(Bucket=bucket_name)
             except ClientError as details:
                 raise StateInaccessibleException(
-                    f"Bucket {bucket_name} is not accessible - {str(details)}"
-                )
+                    f"Bucket {bucket_name} is not accessible - {details!s}"
+                ) from None
 
             # todo: probably remove 'RedHat' from the object/variable/filepath
             # names to keep the code RedHat-agnostic?
@@ -5352,8 +5890,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                 f"from s3 bucket {bucket_name} to local filepath {redhat_logo_png_filepath},"
                 f" but no file exists locally at this path!"
             )
-        file_type = imghdr.what(redhat_logo_png_filepath)
-        if file_type != "png":
+        file_type = filetype.guess(redhat_logo_png_filepath)
+        if file_type.extension != "png":
             raise Exception(
                 f"Attempted to download object {redhat_logo_png_obj_name} "
                 f"from s3 bucket {bucket_name} to local filepath {redhat_logo_png_filepath}."
@@ -5650,7 +6188,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         api_gateway_vpc_link_resource = aws_api_gateway_vpc_link(
             "vpc_link",
             name=f"{identifier}-vpc-link",
-            target_arns=[openshift_ingress_load_balancer_arn]
+            target_arns=[openshift_ingress_load_balancer_arn],
             # future: use data source to get vpc arn by annotation
         )
         tf_resources.append(api_gateway_vpc_link_resource)
@@ -5877,26 +6415,24 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         )
         tf_resources.append(api_gateway_stage_resource)
 
-        rest_api_policy = json.dumps(
-            {
-                "Version": "2012-10-17",
-                "Statement": [
-                    {
-                        "Effect": "Allow",
-                        "Principal": "*",
-                        "Action": "execute-api:Invoke",
-                        "Resource": "${aws_api_gateway_rest_api.gw_api.execution_arn}/*",
-                    },
-                    {
-                        "Effect": "Deny",
-                        "Principal": "*",
-                        "Action": "execute-api:Invoke",
-                        "Resource": "${aws_api_gateway_rest_api.gw_api.execution_arn}/*",
-                        "Condition": {"StringNotEquals": {"aws:SourceVpce": vpce_id}},
-                    },
-                ],
-            }
-        )
+        rest_api_policy = json.dumps({
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": "*",
+                    "Action": "execute-api:Invoke",
+                    "Resource": "${aws_api_gateway_rest_api.gw_api.execution_arn}/*",
+                },
+                {
+                    "Effect": "Deny",
+                    "Principal": "*",
+                    "Action": "execute-api:Invoke",
+                    "Resource": "${aws_api_gateway_rest_api.gw_api.execution_arn}/*",
+                    "Condition": {"StringNotEquals": {"aws:SourceVpce": vpce_id}},
+                },
+            ],
+        })
 
         # REST API POLICY
         api_gateway_rest_api_policy_resource = aws_api_gateway_rest_api_policy(
@@ -5965,6 +6501,7 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     "${aws_cloudwatch_log_group.waf_log_group.arn}"
                 ],
                 resource_arn="${aws_wafv2_web_acl.api_waf.arn}",
+                redacted_fields={"single_header": {"name": "authorization"}},
             )
         )
 
@@ -6120,3 +6657,218 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             tf_resources.append(aws_vpc_endpoint_subnet_association_resource)
 
         self.add_resources(account, tf_resources)
+
+    def populate_tf_resource_msk(self, spec):
+        account = spec.provisioner_name
+        values = self.init_values(spec)
+        output_prefix = spec.output_prefix
+        tf_resources = []
+        resource_id = spec.identifier
+
+        del values["identifier"]
+        values.setdefault("cluster_name", spec.identifier)
+
+        # common
+        self.init_common_outputs(tf_resources, spec)
+
+        # validations
+        if (
+            values["number_of_broker_nodes"]
+            % len(values["broker_node_group_info"]["client_subnets"])
+            != 0
+        ):
+            raise ValueError(
+                "number_of_broker_nodes must be a multiple of the number of specified client subnets."
+            )
+
+        scram_enabled = (
+            values.get("client_authentication", {}).get("sasl", {}).get("scram", False)
+        )
+        scram_users = {}
+        if scram_enabled:
+            if not spec.resource.get("users", []):
+                raise ValueError(
+                    "users attribute must be given when client_authentication.sasl.scram is enabled."
+                )
+            scram_users = {
+                user["name"]: self.secret_reader.read_all(user["secret"])
+                for user in spec.resource["users"]
+            }
+            # validate user objects
+            for user, secret in scram_users.items():
+                if secret.keys() != {"password", "username"}:
+                    raise ValueError(
+                        f"MSK user '{user}' secret must contain only 'username' and 'password' keys!"
+                    )
+
+        # resource - msk config
+        # unique msk config resource name enables "create_before_destroy" lifecycle
+        # which is required when changing version which requires a resource replacement
+        msk_version_str = values["kafka_version"].replace(".", "-")
+        msk_config_name = f"{resource_id}-{msk_version_str}"
+        msk_config = aws_msk_configuration(
+            msk_config_name,
+            name=msk_config_name,
+            kafka_versions=[values["kafka_version"]],
+            server_properties=values["server_properties"],
+            # lifecycle create_before_destroy is required to ensure that the config is created
+            # before it is assigned to the cluster
+            lifecycle={
+                "create_before_destroy": True,
+            },
+        )
+        tf_resources.append(msk_config)
+        values.pop("server_properties", None)
+
+        # resource - cluster
+        values["configuration_info"] = {
+            "arn": "${" + msk_config.arn + "}",
+            "revision": "${" + msk_config.latest_revision + "}",
+        }
+        msk_cluster = aws_msk_cluster(resource_id, **values)
+        tf_resources.append(msk_cluster)
+
+        # resource - cloudwatch
+        if (
+            values.get("logging_info", {})
+            .get("broker_logs", {})
+            .get("cloudwatch_logs", {})
+            .get("enabled", False)
+        ):
+            log_group_values = {
+                "name": f"{resource_id}-msk-broker-logs",
+                "tags": values["tags"],
+                "retention_in_days": values["logging_info"]["broker_logs"][
+                    "cloudwatch_logs"
+                ]["retention_in_days"],
+            }
+            log_group_tf_resource = aws_cloudwatch_log_group(
+                resource_id, **log_group_values
+            )
+            tf_resources.append(log_group_tf_resource)
+            del values["logging_info"]["broker_logs"]["cloudwatch_logs"][
+                "retention_in_days"
+            ]
+            values["logging_info"]["broker_logs"]["cloudwatch_logs"]["log_group"] = (
+                log_group_tf_resource.name
+            )
+
+        # resource - secret manager for SCRAM client credentials
+        if scram_enabled and scram_users:
+            scram_secrets: list[
+                tuple[aws_secretsmanager_secret, aws_secretsmanager_secret_version]
+            ] = []
+
+            # kms
+            kms_values = {
+                "description": "KMS key for MSK SCRAM credentials",
+                "tags": values["tags"],
+            }
+            kms_key = aws_kms_key(resource_id, **kms_values)
+            tf_resources.append(kms_key)
+
+            kms_key_alias = aws_kms_alias(
+                resource_id,
+                name=f"alias/{resource_id}-msk-scram",
+                target_key_id="${" + kms_key.arn + "}",
+            )
+            tf_resources.append(kms_key_alias)
+
+            for user, secret in scram_users.items():
+                secret_identifier = f"AmazonMSK_{resource_id}-{user}"
+
+                secret_values = {
+                    "name": secret_identifier,
+                    "tags": values["tags"],
+                    "kms_key_id": "${" + kms_key.arn + "}",
+                }
+                secret_resource = aws_secretsmanager_secret(
+                    secret_identifier, **secret_values
+                )
+                tf_resources.append(secret_resource)
+
+                version_values = {
+                    "secret_id": "${" + secret_resource.arn + "}",
+                    "secret_string": json.dumps(secret, sort_keys=True),
+                }
+                version_resource = aws_secretsmanager_secret_version(
+                    secret_identifier, **version_values
+                )
+                tf_resources.append(version_resource)
+
+                secret_policy_values = {
+                    "secret_arn": "${" + secret_resource.arn + "}",
+                    "policy": json.dumps({
+                        "Version": "2012-10-17",
+                        "Statement": [
+                            {
+                                "Sid": "AWSKafkaResourcePolicy",
+                                "Effect": "Allow",
+                                "Principal": {"Service": "kafka.amazonaws.com"},
+                                "Action": "secretsmanager:getSecretValue",
+                                "Resource": "${" + secret_resource.arn + "}",
+                            }
+                        ],
+                    }),
+                }
+                secret_policy = aws_secretsmanager_secret_policy(
+                    secret_identifier, **secret_policy_values
+                )
+                tf_resources.append(secret_policy)
+                scram_secrets.append((secret_resource, version_resource))
+
+            # create ONE scram secret association for each secret created above
+            scram_secret_association_values = {
+                "cluster_arn": "${" + msk_cluster.arn + "}",
+                "secret_arn_list": ["${" + s.arn + "}" for s, _ in scram_secrets],
+                "depends_on": self.get_dependencies([v for _, v in scram_secrets]),
+            }
+            scram_secret_association = aws_msk_scram_secret_association(
+                resource_id, **scram_secret_association_values
+            )
+            tf_resources.append(scram_secret_association)
+
+        # outputs
+        tf_resources.append(
+            Output(
+                output_prefix + "__zookeeper_connect_string",
+                value="${" + msk_cluster.zookeeper_connect_string + "}",
+            )
+        )
+        tf_resources.append(
+            Output(
+                output_prefix + "__zookeeper_connect_string_tls",
+                value="${" + msk_cluster.zookeeper_connect_string_tls + "}",
+            )
+        )
+        tf_resources.append(
+            Output(
+                output_prefix + "__bootstrap_brokers",
+                value="${" + msk_cluster.bootstrap_brokers + "}",
+            )
+        )
+        tf_resources.append(
+            Output(
+                output_prefix + "__bootstrap_brokers_tls",
+                value="${" + msk_cluster.bootstrap_brokers_tls + "}",
+            )
+        )
+        tf_resources.append(
+            Output(
+                output_prefix + "__bootstrap_brokers_sasl_iam",
+                value="${" + msk_cluster.bootstrap_brokers_sasl_iam + "}",
+            )
+        )
+        tf_resources.append(
+            Output(
+                output_prefix + "__bootstrap_brokers_sasl_scram",
+                value="${" + msk_cluster.bootstrap_brokers_sasl_scram + "}",
+            )
+        )
+        self.add_resources(account, tf_resources)
+
+    def populate_saml_idp(self, account_name: str, name: str, metadata: str) -> None:
+        saml_idp = aws_iam_saml_provider(
+            f"{account_name}-{name}", name=name, saml_metadata_document=metadata
+        )
+        self.add_resource(account_name, saml_idp)