qontract-reconcile 0.10.2.dev43__py3-none-any.whl → 0.10.2.dev45__py3-none-any.whl

{qontract_reconcile-0.10.2.dev43.dist-info → qontract_reconcile-0.10.2.dev45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev43
+Version: 0.10.2.dev45
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev43.dist-info → qontract_reconcile-0.10.2.dev45.dist-info}/RECORD

@@ -195,11 +195,11 @@ reconcile/endpoints_discovery/integration.py,sha256=fzy5tv4c_6WoAZGGBNJ276NVNB1O
 reconcile/endpoints_discovery/merge_request.py,sha256=_yLb4tnvoZMCko8rta2C_CvOInJa9pa3HzSmHNtjgGU,2978
 reconcile/endpoints_discovery/merge_request_manager.py,sha256=wUMsumxv8RnWaRattax4HfoRlhtVzmgro3GiJJ1C4Vc,6392
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/external_resources/aws.py,sha256=2vimGEtcwcIyVaFvt5Ab2PAAWu2hawJyCu4fuZWSobI,7783
-reconcile/external_resources/factories.py,sha256=ldvm3n13IdE899Ei850NSf0RQ35rzsKJBCUkROrbE5Y,5579
+reconcile/external_resources/aws.py,sha256=KAs656zj4oZYpVbgWsDLXnJWJlLOZdR1JnRhikbF4x0,10712
+reconcile/external_resources/factories.py,sha256=C0QHT0soEv6z99-ELAAE19S5MaMHhV0t1fSiQn0Coc4,5970
 reconcile/external_resources/integration.py,sha256=JF38M7R0Z4ADUTx57TZqSZH9k_xpPlbAxQAcGyIISuM,6925
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
-reconcile/external_resources/manager.py,sha256=DtxjWx34WdPjPR5TzqV4mZpN_Gn20LcNTZHBbPxqzuQ,16953
+reconcile/external_resources/manager.py,sha256=ZagwLn6YQ1XmgmMN3qpuDzQsQxa4VOYl-IQPZBwCDqM,17103
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=KiBjMUaN_z0cSkF_7Ar_a8RiuiwVqjyMcVdISlxhzXE,3898
 reconcile/external_resources/model.py,sha256=EpgIgVRPUsyfHhgjHv_TLUKjzFiIQt0wUd30K0NJJpI,11826
@@ -633,7 +633,7 @@ reconcile/utils/state.py,sha256=az4tBmZ0EdbFcAGiBVUxs3cr2-BVWsuDQiNTvjjQq8s,1637
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
 reconcile/utils/terraform_client.py,sha256=H8frsS370y8xfivKLNBD1dwlBLHvfuR6JSN_syBL5Qc,36033
-reconcile/utils/terrascript_aws_client.py,sha256=SNGtsG1n-IDZaI0blKLm3t3AfVNmxW-O8Y8NtX08OOc,270318
+reconcile/utils/terrascript_aws_client.py,sha256=UdEM3JeTMiE0VRqtz7gcBWR-c0fouORtPFrniRJ3pao,283505
 reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2voKr6KVI34zZE,4833
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/vault.py,sha256=aSA8l9cJlPUHpChFGl27nSY-Mpq9FMjBo7Dcgb1BVfM,15036
@@ -751,7 +751,7 @@ tools/sd_app_sre_alert_report.py,sha256=jQpJdXVID68bSNtJNOGDh0-ei1CfEUS4Itr4MAaB
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/cli_commands/container_images_report.py,sha256=8fG9XU-eEhJ7hKCdQzBcdPpvIJR-8WGkHOgFEulpfYQ,5213
-tools/cli_commands/erv2.py,sha256=iqihq908iJYIQoaNq3Iofp5SQP8S-DhTurzGdnEragU,23218
+tools/cli_commands/erv2.py,sha256=VxUlNXllo947UwmtvS-42IeI9x_t_X3MHrrSI3K_GRo,23274
 tools/cli_commands/gpg_encrypt.py,sha256=NhzwN49UN7P5_FJgTUN5A4BIwNbFokIE4lwDax2iP5k,4891
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
 tools/cli_commands/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -773,7 +773,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev43.dist-info/METADATA,sha256=dR0wcgtr249AUyR4Ry3LQDzF8VgP0FQO4AEs_sCbJLM,24665
-qontract_reconcile-0.10.2.dev43.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev43.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev43.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev45.dist-info/METADATA,sha256=weP8FCL2ss8xcapzIrtxBfdAV3gAjUFV5fhHpfGe3nY,24665
+qontract_reconcile-0.10.2.dev45.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev45.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev45.dist-info/RECORD,,

reconcile/external_resources/aws.py

@@ -1,9 +1,11 @@
+import re
 from abc import ABC, abstractmethod
 from typing import Any
 
 from reconcile.external_resources.model import (
     ExternalResource,
     ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
     ExternalResourcesInventory,
 )
 from reconcile.utils.external_resource_spec import (
@@ -21,10 +23,18 @@ class AWSResourceFactory(ABC):
         self.secret_reader = secret_reader
 
     @abstractmethod
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]: ...
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]: ...
 
     @abstractmethod
-    def validate(self, resource: ExternalResource) -> None: ...
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec
@@ -35,10 +45,18 @@ class AWSResourceFactory(ABC):
 
 
 class AWSDefaultResourceFactory(AWSResourceFactory):
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         return ResourceValueResolver(spec=spec, identifier_as_value=True).resolve()
 
-    def validate(self, resource: ExternalResource) -> None: ...
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...
 
 
 class AWSElasticacheFactory(AWSDefaultResourceFactory):
@@ -49,7 +67,11 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "elasticache", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         """Resolve the elasticache resource specification and translate some attributes to AWS >= 5.60.0 provider format."""
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
@@ -70,7 +92,11 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
 
         return data
 
-    def validate(self, resource: ExternalResource) -> None:
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         """Validate the elasticache resource specification."""
         data = resource.data
         if data.get("parameter_group"):
@@ -96,6 +122,8 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
 
 
 class AWSRdsFactory(AWSDefaultResourceFactory):
+    TIMEOUT_RE = re.compile(r"^\d+m$")
+
     def _get_source_db_spec(
         self, provisioner: str, identifier: str
     ) -> ExternalResourceSpec:
@@ -110,7 +138,11 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "kms", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
 
@@ -126,7 +158,7 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             sourcedb_spec = self._get_source_db_spec(
                 spec.provisioner_name, data["replica_source"]
             )
-            sourcedb = self.resolve(sourcedb_spec)
+            sourcedb = self.resolve(sourcedb_spec, module_conf)
             sourcedb_region = (
                 sourcedb.get("region", None)
                 or sourcedb_spec.provisioner["resources_default_region"]
@@ -142,9 +174,59 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
                 spec.provisioner_name, kms_key_id
             ).identifier
 
+        # If not timeouts are set, set default timeouts according to the module reconcile timeout configuration
+        # 5 minutes are substracted to let terraform finish gracefully before the Job is killed.
+        if "timeouts" not in data:
+            data["timeouts"] = {
+                "create": f"{module_conf.reconcile_timeout_minutes - 5}m",
+                "update": f"{module_conf.reconcile_timeout_minutes - 5}m",
+                "delete": f"{module_conf.reconcile_timeout_minutes - 5}m",
+            }
         return data
 
-    def validate(self, resource: ExternalResource) -> None: ...
+    def _get_timeout_minutes(
+        self,
+        timeout: str,
+    ) -> int:
+        if not re.match(AWSRdsFactory.TIMEOUT_RE, timeout):
+            raise ValueError(
+                f"Invalid RDS instance timeout format: {timeout}. Specify timeout in minutes(m)."
+            )
+        return int(timeout[:-1])
+
+    def _validate_timeouts(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
+        timeouts = resource.data.get("timeouts")
+        if not timeouts:
+            return
+
+        if not isinstance(timeouts, dict):
+            raise ValueError(
+                "Timeouts must be a dictionary with 'create', 'update' and/or 'delete' keys."
+            )
+
+        allowed_keys = {"create", "update", "delete"}
+        if unknown_keys := timeouts.keys() - allowed_keys:
+            raise ValueError(
+                f"Timeouts must be a dictionary with 'create', 'update' and/or 'delete' keys. Offending keys: {unknown_keys}."
+            )
+
+        for option, timeout in timeouts.items():
+            timeout_minutes = self._get_timeout_minutes(timeout)
+            if timeout_minutes >= module_conf.reconcile_timeout_minutes:
+                raise ValueError(
+                    f"RDS instance {option} timeout value {timeout_minutes} must be lower than the module reconcile_timeout_minutes value {module_conf.reconcile_timeout_minutes}."
+                )
+
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
+        self._validate_timeouts(resource, module_conf)
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec
@@ -166,7 +248,11 @@ class AWSMskFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "msk", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
         data["output_prefix"] = spec.output_prefix
@@ -188,7 +274,11 @@ class AWSMskFactory(AWSDefaultResourceFactory):
             del data["users"]
         return data
 
-    def validate(self, resource: ExternalResource) -> None:
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         data = resource.data
         if (
             data["number_of_broker_nodes"]

reconcile/external_resources/factories.py

@@ -15,6 +15,7 @@ from reconcile.external_resources.meta import QONTRACT_INTEGRATION
 from reconcile.external_resources.model import (
     ExternalResource,
     ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
     ExternalResourceProvision,
     ExternalResourcesInventory,
     ModuleInventory,
@@ -55,11 +56,19 @@ class ObjectFactory(Generic[T]):
 
 class ExternalResourceFactory(ABC):
     @abstractmethod
-    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def create_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         pass
 
     @abstractmethod
-    def validate_external_resource(self, resource: ExternalResource) -> None:
+    def validate_external_resource(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         pass
 
     def find_linked_resources(
@@ -121,9 +130,13 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
         self.er_inventory = er_inventory
         self.secret_reader = secret_reader
 
-    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def create_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         f = self.resource_factories.get_factory(spec.provider)
-        data = f.resolve(spec)
+        data = f.resolve(spec, module_conf)
         data["tags"] = spec.tags(integration=QONTRACT_INTEGRATION)
         data["default_tags"] = AWS_DEFAULT_TAGS
 
@@ -152,9 +165,13 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
 
         return ExternalResource(data=data, provision=provision)
 
-    def validate_external_resource(self, resource: ExternalResource) -> None:
+    def validate_external_resource(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         f = self.resource_factories.get_factory(resource.provision.provider)
-        f.validate(resource)
+        f.validate(resource, module_conf)
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec

reconcile/external_resources/manager.py

@@ -197,8 +197,11 @@ class ExternalResourcesManager:
             if spec.marked_to_delete:
                 continue
             module = self.module_inventory.get_from_spec(spec)
+            module_conf = ExternalResourceModuleConfiguration.resolve_configuration(
+                module, spec, self.settings
+            )
             try:
-                resource = self._build_external_resource(spec)
+                resource = self._build_external_resource(spec, module_conf)
             except ExternalResourceValidationError as e:
                 self.errors[key] = e
                 continue
@@ -208,9 +211,7 @@ class ExternalResourcesManager:
                 resource_hash=resource.hash(),
                 input=resource.json(),
                 action=Action.APPLY,
-                module_configuration=ExternalResourceModuleConfiguration.resolve_configuration(
-                    module, spec, self.settings
-                ),
+                module_configuration=module_conf,
                 linked_resources=self._find_linked_resources(spec),
             )
             r.add(reconciliation)
@@ -362,10 +363,14 @@ class ExternalResourcesManager:
                 )
                 self.state_mgr.update_resource_status(key, ResourceStatus.CREATED)
 
-    def _build_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def _build_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         f = self.factories.get_factory(spec.provision_provider)
-        resource = f.create_external_resource(spec)
-        f.validate_external_resource(resource)
+        resource = f.create_external_resource(spec, module_conf)
+        f.validate_external_resource(resource, module_conf)
         return resource
 
     def _find_linked_resources(

reconcile/utils/terrascript_aws_client.py

@@ -77,6 +77,8 @@ from terrascript.resource import (
     aws_ec2_transit_gateway_vpc_attachment,
     aws_ec2_transit_gateway_vpc_attachment_accepter,
     aws_ecr_repository,
+    aws_elasticache_parameter_group,
+    aws_elasticache_replication_group,
     aws_elasticsearch_domain,
     aws_iam_access_key,
     aws_iam_group,
@@ -104,6 +106,8 @@ from terrascript.resource import (
     aws_lb_listener_rule,
     aws_lb_target_group,
     aws_lb_target_group_attachment,
+    aws_msk_cluster,
+    aws_msk_configuration,
     aws_ram_principal_association,
     aws_ram_resource_association,
     aws_ram_resource_share,
@@ -1641,6 +1645,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.populate_tf_resource_rds(spec)
         elif provider == "s3":
             self.populate_tf_resource_s3(spec)
+        elif provider == "elasticache":
+            self.populate_tf_resource_elasticache(spec)
         elif provider == "aws-iam-service-account":
             self.populate_tf_resource_service_account(spec, ocm_map=ocm_map)
         elif provider == "secrets-manager-service-account":
@@ -1683,6 +1689,8 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.populate_tf_resource_rosa_authenticator(spec)
         elif provider == "rosa-authenticator-vpce":
             self.populate_tf_resource_rosa_authenticator_vpce(spec)
+        elif provider == "msk":
+            self.populate_tf_resource_msk(spec)
         else:
             raise UnknownProviderError(provider)
 
@@ -2483,6 +2491,94 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         return bucket_tf_resource
 
+    def populate_tf_resource_elasticache(self, spec):
+        account = spec.provisioner_name
+        identifier = spec.identifier
+        values = self.init_values(spec)
+        output_prefix = spec.output_prefix
+        values.setdefault("replication_group_id", values["identifier"])
+        values.pop("identifier", None)
+
+        tf_resources = []
+        self.init_common_outputs(tf_resources, spec)
+
+        default_region = self.default_regions.get(account)
+        desired_region = values.pop("region", default_region)
+
+        provider = ""
+        if desired_region is not None and self._multiregion_account(account):
+            provider = "aws." + desired_region
+            values["provider"] = provider
+
+        if not values.get("apply_immediately"):
+            values["apply_immediately"] = False
+
+        parameter_group = values.get("parameter_group")
+        # Assume that cluster enabled is false if parameter group unset
+        pg_cluster_enabled = False
+
+        if parameter_group:
+            pg_values = self.get_values(parameter_group)
+            pg_name = pg_values["name"]
+            pg_identifier = pg_name
+
+            # If the desired region is not the same as the default region
+            # we append the region to the identifier to make it unique
+            # in the terraform config
+            if desired_region is not None and desired_region != default_region:
+                pg_identifier = f"{pg_name}-{desired_region}"
+
+            pg_values["parameter"] = pg_values.pop("parameters")
+            for param in pg_values["parameter"]:
+                if param["name"] == "cluster-enabled" and param["value"] == "yes":
+                    pg_cluster_enabled = True
+
+            if self._multiregion_account(account) and len(provider) > 0:
+                pg_values["provider"] = provider
+            pg_tf_resource = aws_elasticache_parameter_group(pg_identifier, **pg_values)
+            tf_resources.append(pg_tf_resource)
+            values["depends_on"] = [
+                f"aws_elasticache_parameter_group.{pg_identifier}",
+            ]
+            values["parameter_group_name"] = pg_name
+            values.pop("parameter_group", None)
+
+        auth_token = spec.get_secret_field("db.auth_token")
+        if not auth_token:
+            auth_token = self.generate_random_password()
+
+        if values.get("transit_encryption_enabled", False):
+            values["auth_token"] = auth_token
+
+        # elasticache replication group
+        # Ref: https://www.terraform.io/docs/providers/aws/r/
+        # elasticache_replication_group.html
+        tf_resource = aws_elasticache_replication_group(identifier, **values)
+        tf_resources.append(tf_resource)
+        # elasticache outputs
+        # we want the outputs to be formed into an OpenShift Secret
+        # with the following fields
+        # db.endpoint
+        output_name = output_prefix + "__db_endpoint"
+        # https://docs.aws.amazon.com/AmazonElastiCache/
+        # latest/red-ug/Endpoints.html
+        if pg_cluster_enabled:
+            output_value = "${" + tf_resource.configuration_endpoint_address + "}"
+        else:
+            output_value = "${" + tf_resource.primary_endpoint_address + "}"
+        tf_resources.append(Output(output_name, value=output_value))
+        # db.port
+        output_name = output_prefix + "__db_port"
+        output_value = "${" + str(tf_resource.port) + "}"
+        tf_resources.append(Output(output_name, value=output_value))
+        # db.auth_token
+        if values.get("transit_encryption_enabled", False):
+            output_name = output_prefix + "__db_auth_token"
+            output_value = values["auth_token"]
+            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
+
+        self.add_resources(account, tf_resources)
+
     def populate_tf_resource_service_account(self, spec, ocm_map=None):
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -4191,6 +4287,24 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     values["db_name"] = db_name
                 if values.get("replica_source"):
                     values.pop("db_name", None)
+            elif spec.provider == "elasticache":
+                if description := values.pop("replication_group_description", None):
+                    values["description"] = description
+                if num_cache_clusters := values.pop("number_cache_clusters", None):
+                    values["num_cache_clusters"] = num_cache_clusters
+                if cluster_mode := values.pop("cluster_mode", {}):
+                    for k, v in cluster_mode.items():
+                        values[k] = v
+                values.pop("availability_zones", None)
+            elif spec.provider == "msk":
+                if ebs_volume_size := values.get("broker_node_group_info", {}).pop(
+                    "ebs_volume_size", None
+                ):
+                    values["broker_node_group_info"].setdefault(
+                        "storage_info", {}
+                    ).setdefault("ebs_storage_info", {})[
+                        "volume_size"
+                    ] = ebs_volume_size
 
         return values
 
@@ -6542,6 +6656,205 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         self.add_resources(account, tf_resources)
 
+    def populate_tf_resource_msk(self, spec):
+        account = spec.provisioner_name
+        values = self.init_values(spec)
+        output_prefix = spec.output_prefix
+        tf_resources = []
+        resource_id = spec.identifier
+
+        del values["identifier"]
+        values.setdefault("cluster_name", spec.identifier)
+
+        # common
+        self.init_common_outputs(tf_resources, spec)
+
+        # validations
+        if (
+            values["number_of_broker_nodes"]
+            % len(values["broker_node_group_info"]["client_subnets"])
+            != 0
+        ):
+            raise ValueError(
+                "number_of_broker_nodes must be a multiple of the number of specified client subnets."
+            )
+
+        scram_enabled = (
+            values.get("client_authentication", {}).get("sasl", {}).get("scram", False)
+        )
+        scram_users = {}
+        if scram_enabled:
+            if not spec.resource.get("users", []):
+                raise ValueError(
+                    "users attribute must be given when client_authentication.sasl.scram is enabled."
+                )
+            scram_users = {
+                user["name"]: self.secret_reader.read_all(user["secret"])
+                for user in spec.resource["users"]
+            }
+            # validate user objects
+            for user, secret in scram_users.items():
+                if secret.keys() != {"password", "username"}:
+                    raise ValueError(
+                        f"MSK user '{user}' secret must contain only 'username' and 'password' keys!"
+                    )
+
+        # resource - msk config
+        # unique msk config resource name enables "create_before_destroy" lifecycle
+        # which is required when changing version which requires a resource replacement
+        msk_version_str = values["kafka_version"].replace(".", "-")
+        msk_config_name = f"{resource_id}-{msk_version_str}"
+        msk_config = aws_msk_configuration(
+            msk_config_name,
+            name=msk_config_name,
+            kafka_versions=[values["kafka_version"]],
+            server_properties=values["server_properties"],
+            # lifecycle create_before_destroy is required to ensure that the config is created
+            # before it is assigned to the cluster
+            lifecycle={
+                "create_before_destroy": True,
+            },
+        )
+        tf_resources.append(msk_config)
+        values.pop("server_properties", None)
+
+        # resource - cluster
+        values["configuration_info"] = {
+            "arn": "${" + msk_config.arn + "}",
+            "revision": "${" + msk_config.latest_revision + "}",
+        }
+        msk_cluster = aws_msk_cluster(resource_id, **values)
+        tf_resources.append(msk_cluster)
+
+        # resource - cloudwatch
+        if (
+            values.get("logging_info", {})
+            .get("broker_logs", {})
+            .get("cloudwatch_logs", {})
+            .get("enabled", False)
+        ):
+            log_group_values = {
+                "name": f"{resource_id}-msk-broker-logs",
+                "tags": values["tags"],
+                "retention_in_days": values["logging_info"]["broker_logs"][
+                    "cloudwatch_logs"
+                ]["retention_in_days"],
+            }
+            log_group_tf_resource = aws_cloudwatch_log_group(
+                resource_id, **log_group_values
+            )
+            tf_resources.append(log_group_tf_resource)
+            del values["logging_info"]["broker_logs"]["cloudwatch_logs"][
+                "retention_in_days"
+            ]
+            values["logging_info"]["broker_logs"]["cloudwatch_logs"]["log_group"] = (
+                log_group_tf_resource.name
+            )
+
+        # resource - secret manager for SCRAM client credentials
+        if scram_enabled and scram_users:
+            scram_secrets: list[
+                tuple[aws_secretsmanager_secret, aws_secretsmanager_secret_version]
+            ] = []
+
+            # kms
+            kms_values = {
+                "description": "KMS key for MSK SCRAM credentials",
+                "tags": values["tags"],
+            }
+            kms_key = aws_kms_key(resource_id, **kms_values)
+            tf_resources.append(kms_key)
+
+            kms_key_alias = aws_kms_alias(
+                resource_id,
+                name=f"alias/{resource_id}-msk-scram",
+                target_key_id="${" + kms_key.arn + "}",
+            )
+            tf_resources.append(kms_key_alias)
+
+            for user, secret in scram_users.items():
+                secret_identifier = f"AmazonMSK_{resource_id}-{user}"
+
+                secret_values = {
+                    "name": secret_identifier,
+                    "tags": values["tags"],
+                    "kms_key_id": "${" + kms_key.arn + "}",
+                }
+                secret_resource = aws_secretsmanager_secret(
+                    secret_identifier, **secret_values
+                )
+                tf_resources.append(secret_resource)
+
+                version_values = {
+                    "secret_id": "${" + secret_resource.arn + "}",
+                    "secret_string": json.dumps(secret, sort_keys=True),
+                }
+                version_resource = aws_secretsmanager_secret_version(
+                    secret_identifier, **version_values
+                )
+                tf_resources.append(version_resource)
+
+                secret_policy_values = {
+                    "secret_arn": "${" + secret_resource.arn + "}",
+                    "policy": json.dumps({
+                        "Version": "2012-10-17",
+                        "Statement": [
+                            {
+                                "Sid": "AWSKafkaResourcePolicy",
+                                "Effect": "Allow",
+                                "Principal": {"Service": "kafka.amazonaws.com"},
+                                "Action": "secretsmanager:getSecretValue",
+                                "Resource": "${" + secret_resource.arn + "}",
+                            }
+                        ],
+                    }),
+                }
+                secret_policy = aws_secretsmanager_secret_policy(
+                    secret_identifier, **secret_policy_values
+                )
+                tf_resources.append(secret_policy)
+                scram_secrets.append((secret_resource, version_resource))
+
+            # create ONE scram secret association for each secret created above
+            scram_secret_association_values = {
+                "cluster_arn": "${" + msk_cluster.arn + "}",
+                "secret_arn_list": ["${" + s.arn + "}" for s, _ in scram_secrets],
+                "depends_on": self.get_dependencies([v for _, v in scram_secrets]),
+            }
+            scram_secret_association = aws_msk_scram_secret_association(
+                resource_id, **scram_secret_association_values
+            )
+            tf_resources.append(scram_secret_association)
+
+        # outputs
+        tf_resources += [
+            Output(
+                output_prefix + "__zookeeper_connect_string",
+                value="${" + msk_cluster.zookeeper_connect_string + "}",
+            ),
+            Output(
+                output_prefix + "__zookeeper_connect_string_tls",
+                value="${" + msk_cluster.zookeeper_connect_string_tls + "}",
+            ),
+            Output(
+                output_prefix + "__bootstrap_brokers",
+                value="${" + msk_cluster.bootstrap_brokers + "}",
+            ),
+            Output(
+                output_prefix + "__bootstrap_brokers_tls",
+                value="${" + msk_cluster.bootstrap_brokers_tls + "}",
+            ),
+            Output(
+                output_prefix + "__bootstrap_brokers_sasl_iam",
+                value="${" + msk_cluster.bootstrap_brokers_sasl_iam + "}",
+            ),
+            Output(
+                output_prefix + "__bootstrap_brokers_sasl_scram",
+                value="${" + msk_cluster.bootstrap_brokers_sasl_scram + "}",
+            ),
+        ]
+        self.add_resources(account, tf_resources)
+
     def populate_saml_idp(self, account_name: str, name: str, metadata: str) -> None:
         saml_idp = aws_iam_saml_provider(
             f"{account_name}-{name}", name=name, saml_metadata_document=metadata

tools/cli_commands/erv2.py

@@ -118,13 +118,13 @@ class Erv2Cli:
             self._er_settings, m_inventory, er_inventory, self._secret_reader
         )
         f = factories.get_factory(spec.provision_provider)
-        self._resource = f.create_external_resource(spec)
-        f.validate_external_resource(self._resource)
         self._module_configuration = (
             ExternalResourceModuleConfiguration.resolve_configuration(
                 m_inventory.get_from_spec(spec), spec, self._er_settings
             )
         )
+        self._resource = f.create_external_resource(spec, self._module_configuration)
+        f.validate_external_resource(self._resource, self._module_configuration)
 
     @property
     def input_data(self) -> str: