qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/terrascript_aws_client.py

@@ -22,7 +22,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Self,
-    TypeAlias,
     cast,
 )
 
@@ -152,6 +151,7 @@ from reconcile.github_org import get_default_config
 from reconcile.gql_definitions.terraform_resources.terraform_resources_namespaces import (
     NamespaceTerraformResourceLifecycleV1,
 )
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
 from reconcile.utils import gql
 from reconcile.utils.aws_api import (
     AmiTag,
@@ -178,7 +178,11 @@ from reconcile.utils.external_resources import (
 from reconcile.utils.git import is_file_in_git_repo
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jenkins_api import JenkinsApi
-from reconcile.utils.jinja2.utils import process_extracurlyjinja2_template
+from reconcile.utils.jinja2.utils import (
+    process_extracurlyjinja2_template,
+    process_jinja2_template,
+)
+from reconcile.utils.json import json_dumps
 from reconcile.utils.password_validator import (
     PasswordPolicy,
     PasswordValidator,
@@ -202,7 +206,7 @@ if TYPE_CHECKING:
     from reconcile.utils.ocm import OCMMap
 
 
-TFResource: TypeAlias = type[
+type TFResource = type[
     Resource | Data | Module | Provider | Variable | Output | Locals | Terraform
 ]
 
@@ -267,6 +271,8 @@ VARIABLE_KEYS = [
     "extra_tags",
     "lifecycle",
     "max_session_duration",
+    "secret_format",
+    "policy",
 ]
 
 EMAIL_REGEX = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")
@@ -282,15 +288,10 @@ SUPPORTED_ALB_LISTENER_RULE_CONDITION_TYPE_MAPPING = {
     "host-header": "host_header",
     "http-request-method": "http_request_method",
     "path-pattern": "path_pattern",
+    "query-string": "query_string",
     "source-ip": "source_ip",
 }
 
-DEFAULT_TAGS = {
-    "tags": {
-        "app": "app-sre-infra",
-    },
-}
-
 AWS_ELB_ACCOUNT_IDS = {
     "us-east-1": "127311923021",
     "us-east-2": "033677994240",
@@ -373,6 +374,10 @@ class aws_s3_bucket_logging(Resource):
     pass
 
 
+class aws_kinesis_resource_policy(Resource):
+    pass
+
+
 class aws_cloudfront_log_delivery_canonical_user_id(Data):
     pass
 
@@ -474,6 +479,7 @@ class TerrascriptClient:
         integration_prefix: str,
         thread_pool_size: int,
         accounts: Iterable[MutableMapping[str, Any]],
+        default_tags: Mapping[str, str] | None,
         settings: Mapping[str, Any] | None = None,
         prefetch_resources_by_schemas: Iterable[str] | None = None,
         secret_reader: SecretReaderBase | None = None,
@@ -487,6 +493,7 @@ class TerrascriptClient:
         else:
             self.secret_reader = SecretReader(settings=settings)
         self.configs: dict[str, dict] = {}
+        self.default_tags = default_tags or {"app": "app-sre-infra"}
         self.populate_configs(filtered_accounts)
         self.versions: dict[str, str] = {
             a["name"]: a["providerVersion"] for a in filtered_accounts
@@ -507,7 +514,7 @@ class TerrascriptClient:
                         region=region,
                         alias=region,
                         skip_region_validation=True,
-                        default_tags=DEFAULT_TAGS,
+                        default_tags={"tags": config["tags"]},
                     )
 
             # Add default region, which will be in resourcesDefaultRegion
@@ -516,7 +523,7 @@ class TerrascriptClient:
                 secret_key=config["aws_secret_access_key"],
                 region=config["resourcesDefaultRegion"],
                 skip_region_validation=True,
-                default_tags=DEFAULT_TAGS,
+                default_tags={"tags": config["tags"]},
             )
 
             ts += Terraform(
@@ -799,6 +806,9 @@ class TerrascriptClient:
             config["supportedDeploymentRegions"] = account["supportedDeploymentRegions"]
             config["resourcesDefaultRegion"] = account["resourcesDefaultRegion"]
             config["terraformState"] = account["terraformState"]
+            config["tags"] = dict(self.default_tags) | get_aws_account_tags(
+                account.get("organization", None)
+            )
             self.configs[account_name] = config
 
     def _get_partition(self, account: str) -> str:
@@ -1053,7 +1063,9 @@ class TerrascriptClient:
             ignore_changes = (
                 "all" if "all" in lifecycle.ignore_changes else lifecycle.ignore_changes
             )
-            return lifecycle.dict(by_alias=True) | {"ignore_changes": ignore_changes}
+            return lifecycle.model_dump(by_alias=True) | {
+                "ignore_changes": ignore_changes
+            }
         return None
 
     def populate_additional_providers(
@@ -1068,25 +1080,15 @@ class TerrascriptClient:
             config = self.configs[account_name]
             existing_provider_aliases = {p.get("alias") for p in ts["provider"]["aws"]}
             if alias not in existing_provider_aliases:
-                if assume_role:
-                    ts += provider.aws(
-                        access_key=config["aws_access_key_id"],
-                        secret_key=config["aws_secret_access_key"],
-                        region=region,
-                        alias=alias,
-                        assume_role={"role_arn": assume_role},
-                        skip_region_validation=True,
-                        default_tags=DEFAULT_TAGS,
-                    )
-                else:
-                    ts += provider.aws(
-                        access_key=config["aws_access_key_id"],
-                        secret_key=config["aws_secret_access_key"],
-                        region=region,
-                        alias=alias,
-                        skip_region_validation=True,
-                        default_tags=DEFAULT_TAGS,
-                    )
+                ts += provider.aws(
+                    access_key=config["aws_access_key_id"],
+                    secret_key=config["aws_secret_access_key"],
+                    region=region,
+                    alias=alias,
+                    skip_region_validation=True,
+                    default_tags={"tags": config["tags"]},
+                    **{"assume_role": {"role_arn": assume_role}} if assume_role else {},
+                )
 
     def populate_route53(
         self, desired_state: Iterable[Mapping[str, Any]], default_ttl: int = 300
@@ -1343,13 +1345,13 @@ class TerrascriptClient:
                     vpc_module_values["public_subnets"] = request.subnets.public
                     vpc_module_values["public_subnet_tags"] = (
                         VPC_REQUEST_DEFAULT_PUBLIC_SUBNET_TAGS
-                        | (request.subnets.public_subnet_tags or {}),
+                        | (request.subnets.public_subnet_tags or {})
                     )
                 if request.subnets.private:
                     vpc_module_values["private_subnets"] = request.subnets.private
                     vpc_module_values["private_subnet_tags"] = (
                         VPC_REQUEST_DEFAULT_PRIVATE_SUBNET_TAGS
-                        | (request.subnets.private_subnet_tags or {}),
+                        | (request.subnets.private_subnet_tags or {})
                     )
                 if request.subnets.availability_zones:
                     vpc_module_values["azs"] = request.subnets.availability_zones
@@ -1429,7 +1431,7 @@ class TerrascriptClient:
             req_account_name = req_account.name
             # Accepter's side of the connection - the cluster's account
             acc_account = accepter.account
-            acc_alias = self.get_provider_alias(acc_account.dict(by_alias=True))
+            acc_alias = self.get_provider_alias(acc_account.model_dump(by_alias=True))
             acc_uid = acc_account.uid
             if acc_account.assume_role:
                 acc_uid = awsh.get_account_uid_from_arn(acc_account.assume_role)
@@ -1946,7 +1948,7 @@ class TerrascriptClient:
             em_identifier = f"{identifier}-enhanced-monitoring"
             em_values = {
                 "name": em_identifier,
-                "assume_role_policy": json.dumps(assume_role_policy, sort_keys=True),
+                "assume_role_policy": json_dumps(assume_role_policy),
             }
             role_tf_resource = aws_iam_role(em_identifier, **em_values)
             tf_resources.append(role_tf_resource)
@@ -2215,6 +2217,43 @@ class TerrascriptClient:
         letters_and_digits = string.ascii_letters + string.digits
         return "".join(random.choice(letters_and_digits) for i in range(string_length))
 
+    @staticmethod
+    def _build_tf_resource_s3_lifecycle_rules(
+        versioning: bool,
+        common_values: Mapping[str, Any],
+    ) -> list[dict]:
+        lifecycle_rules = common_values.get("lifecycle_rules") or []
+        if versioning and not any(
+            "noncurrent_version_expiration" in lr for lr in lifecycle_rules
+        ):
+            # Add a default noncurrent object expiration rule
+            # if one isn't already set
+            rule = {
+                "id": "expire_noncurrent_versions",
+                "enabled": True,
+                "noncurrent_version_expiration": {"days": 30},
+                "expiration": {"expired_object_delete_marker": True},
+                "abort_incomplete_multipart_upload_days": 3,
+            }
+            lifecycle_rules.append(rule)
+
+        if storage_class := common_values.get("storage_class"):
+            sc = storage_class.upper()
+            days = "1"
+            if sc.endswith("_IA"):
+                # Infrequent Access storage class has minimum 30 days
+                # before transition
+                days = "30"
+            rule = {
+                "id": sc + "_storage_class",
+                "enabled": True,
+                "transition": {"days": days, "storage_class": sc},
+                "noncurrent_version_transition": {"days": days, "storage_class": sc},
+            }
+            lifecycle_rules.append(rule)
+
+        return lifecycle_rules
+
     def populate_tf_resource_s3(self, spec: ExternalResourceSpec) -> aws_s3_bucket:
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -2254,47 +2293,11 @@ class TerrascriptClient:
         request_payer = common_values.get("request_payer")
         if request_payer:
             values["request_payer"] = request_payer
-        lifecycle_rules = common_values.get("lifecycle_rules")
-        if lifecycle_rules:
-            # common_values['lifecycle_rules'] is a list of lifecycle_rules
+        if lifecycle_rules := self._build_tf_resource_s3_lifecycle_rules(
+            versioning=versioning,
+            common_values=common_values,
+        ):
             values["lifecycle_rule"] = lifecycle_rules
-        if versioning:
-            lrs = values.get("lifecycle_rule", [])
-            expiration_rule = False
-            for lr in lrs:
-                if "noncurrent_version_expiration" in lr:
-                    expiration_rule = True
-                    break
-            if not expiration_rule:
-                # Add a default noncurrent object expiration rule if
-                # if one isn't already set
-                rule = {
-                    "id": "expire_noncurrent_versions",
-                    "enabled": "true",
-                    "noncurrent_version_expiration": {"days": 30},
-                }
-                if len(lrs) > 0:
-                    lrs.append(rule)
-                else:
-                    lrs = rule
-        sc = common_values.get("storage_class")
-        if sc:
-            sc = sc.upper()
-            days = "1"
-            if sc.endswith("_IA"):
-                # Infrequent Access storage class has minimum 30 days
-                # before transition
-                days = "30"
-            rule = {
-                "id": sc + "_storage_class",
-                "enabled": "true",
-                "transition": {"days": days, "storage_class": sc},
-                "noncurrent_version_transition": {"days": days, "storage_class": sc},
-            }
-            if values.get("lifecycle_rule"):
-                values["lifecycle_rule"].append(rule)
-            else:
-                values["lifecycle_rule"] = rule
         cors_rules = common_values.get("cors_rules")
         if cors_rules:
             # common_values['cors_rules'] is a list of cors_rules
@@ -2344,7 +2347,7 @@ class TerrascriptClient:
                         }
                     ],
                 }
-                rc_values["assume_role_policy"] = json.dumps(role, sort_keys=True)
+                rc_values["assume_role_policy"] = json_dumps(role)
                 role_resource = aws_iam_role(id, **rc_values)
                 tf_resources.append(role_resource)
 
@@ -2382,7 +2385,7 @@ class TerrascriptClient:
                         },
                     ],
                 }
-                rc_values["policy"] = json.dumps(policy, sort_keys=True)
+                rc_values["policy"] = json_dumps(policy)
                 policy_resource = aws_iam_policy(id, **rc_values)
                 tf_resources.append(policy_resource)
 
@@ -2597,7 +2600,7 @@ class TerrascriptClient:
                 },
             ],
         }
-        values["policy"] = json.dumps(policy, sort_keys=True)
+        values["policy"] = json_dumps(policy)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
@@ -2876,7 +2879,7 @@ class TerrascriptClient:
         values: dict[str, Any] = {
             "name": identifier,
             "tags": common_values["tags"],
-            "assume_role_policy": json.dumps(assume_role_policy),
+            "assume_role_policy": json_dumps(assume_role_policy),
         }
 
         inline_policy = common_values.get("inline_policy")
@@ -2930,7 +2933,7 @@ class TerrascriptClient:
         self, account: str, name: str, policy: Mapping[str, Any]
     ) -> None:
         tf_aws_iam_policy = aws_iam_policy(
-            f"{account}-{name}", name=name, policy=json.dumps(policy)
+            f"{account}-{name}", name=name, policy=json_dumps(policy)
         )
         self.add_resource(account, tf_aws_iam_policy)
 
@@ -2972,7 +2975,7 @@ class TerrascriptClient:
         role_tf_resource = aws_iam_role(
             f"{account}-{name}",
             name=name,
-            assume_role_policy=json.dumps(assume_role_policy),
+            assume_role_policy=json_dumps(assume_role_policy),
             managed_policy_arns=managed_policy_arns,
             max_session_duration=max_session_duration_hours * 3600,
         )
@@ -3016,7 +3019,7 @@ class TerrascriptClient:
                 all_queues.append(queue_name)
                 sqs_policy = values.pop("sqs_policy", None)
                 if sqs_policy is not None:
-                    values["policy"] = json.dumps(sqs_policy, sort_keys=True)
+                    values["policy"] = json_dumps(sqs_policy)
                 dl_queue = values.pop("dl_queue", None)
                 if dl_queue is not None:
                     max_receive_count = int(values.pop("max_receive_count", 10))
@@ -3030,9 +3033,7 @@ class TerrascriptClient:
                         "deadLetterTargetArn": "${" + dl_data.arn + "}",
                         "maxReceiveCount": max_receive_count,
                     }
-                    values["redrive_policy"] = json.dumps(
-                        redrive_policy, sort_keys=True
-                    )
+                    values["redrive_policy"] = json_dumps(redrive_policy)
                 kms_master_key_id = values.pop("kms_master_key_id", None)
                 if kms_master_key_id is not None:
                     if kms_master_key_id.startswith("arn:"):
@@ -3105,7 +3106,7 @@ class TerrascriptClient:
                     "Resource": list(kms_keys),
                 }
                 policy["Statement"].append(kms_statement)
-            values["policy"] = json.dumps(policy, sort_keys=True)
+            values["policy"] = json_dumps(policy)
             policy_tf_resource = aws_iam_policy(policy_identifier, **values)
             tf_resources.append(policy_tf_resource)
 
@@ -3255,7 +3256,7 @@ class TerrascriptClient:
                 }
             ],
         }
-        values["policy"] = json.dumps(policy, sort_keys=True)
+        values["policy"] = json_dumps(policy)
         values["depends_on"] = self.get_dependencies([user_tf_resource])
 
         tf_aws_iam_policy = aws_iam_policy(identifier, **values)
@@ -3365,7 +3366,7 @@ class TerrascriptClient:
                 },
             ],
         }
-        values_policy["policy"] = json.dumps(policy, sort_keys=True)
+        values_policy["policy"] = json_dumps(policy)
         values_policy["depends_on"] = self.get_dependencies([user_tf_resource])
 
         tf_aws_iam_policy = aws_iam_policy(identifier, **values_policy)
@@ -3415,7 +3416,7 @@ class TerrascriptClient:
                 }
             ],
         }
-        values_policy["policy"] = json.dumps(policy, sort_keys=True)
+        values_policy["policy"] = json_dumps(policy)
         values_policy["depends_on"] = self.get_dependencies([bucket_tf_resource])
         region = common_values.get("region") or self.default_regions.get(account)
         assert region  # make mypy happy
@@ -3561,7 +3562,7 @@ class TerrascriptClient:
                 }
             ],
         }
-        sqs_values["policy"] = json.dumps(sqs_policy, sort_keys=True)
+        sqs_values["policy"] = json_dumps(sqs_policy)
 
         kms_encryption = common_values.get("kms_encryption", False)
         if kms_encryption:
@@ -3597,7 +3598,7 @@ class TerrascriptClient:
                     },
                 ],
             }
-            kms_values["policy"] = json.dumps(kms_policy, sort_keys=True)
+            kms_values["policy"] = json_dumps(kms_policy)
             if provider:
                 kms_values["provider"] = provider
 
@@ -3695,7 +3696,7 @@ class TerrascriptClient:
                 "Resource": [sqs_values["kms_master_key_id"]],
             }
             policy["Statement"].append(kms_statement)
-        values_policy["policy"] = json.dumps(policy, sort_keys=True)
+        values_policy["policy"] = json_dumps(policy)
         policy_tf_resource = aws_iam_policy(sqs_identifier, **values_policy)
         tf_resources.append(policy_tf_resource)
 
@@ -3766,7 +3767,7 @@ class TerrascriptClient:
             role_identifier = f"{identifier}-lambda-execution-role"
             role_values = {
                 "name": role_identifier,
-                "assume_role_policy": json.dumps(assume_role_policy, sort_keys=True),
+                "assume_role_policy": json_dumps(assume_role_policy),
             }
 
             role_tf_resource = aws_iam_role(role_identifier, **role_values)
@@ -3798,7 +3799,7 @@ class TerrascriptClient:
 
             policy_values = {
                 "role": "${" + role_tf_resource.id + "}",
-                "policy": json.dumps(policy, sort_keys=True),
+                "policy": json_dumps(policy),
             }
             policy_tf_resource = aws_iam_role_policy(policy_identifier, **policy_values)
             tf_resources.append(policy_tf_resource)
@@ -3926,7 +3927,7 @@ class TerrascriptClient:
         }
         values = {
             "name": identifier,
-            "policy": json.dumps(policy, sort_keys=True),
+            "policy": json_dumps(policy),
             "depends_on": self.get_dependencies([user_tf_resource]),
         }
 
@@ -4023,6 +4024,22 @@ class TerrascriptClient:
         kinesis_tf_resource = aws_kinesis_stream(identifier, **kinesis_values)
         tf_resources.append(kinesis_tf_resource)
 
+        # kinesis resource policy (optional)
+        # Terraform resource reference:
+        # https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_resource_policy
+        if policy := common_values.get("policy"):
+            policy_identifier = f"{identifier}-policy"
+            policy_values: dict[str, Any] = {
+                "resource_arn": "${" + kinesis_tf_resource.arn + "}",
+                "policy": policy,
+            }
+            if provider:
+                policy_values["provider"] = provider
+            kinesis_policy_tf_resource = aws_kinesis_resource_policy(
+                policy_identifier, **policy_values
+            )
+            tf_resources.append(kinesis_policy_tf_resource)
+
         es_identifier = common_values.get("es_identifier", None)
         if es_identifier:
             es_resource = self._find_resource_spec(
@@ -4047,7 +4064,7 @@ class TerrascriptClient:
             role_identifier = f"{identifier}-lambda-execution-role"
             role_values = {
                 "name": role_identifier,
-                "assume_role_policy": json.dumps(assume_role_policy, sort_keys=True),
+                "assume_role_policy": json_dumps(assume_role_policy),
                 "tags": tags,
             }
 
@@ -4084,7 +4101,7 @@ class TerrascriptClient:
             policy_tf_resource = aws_iam_policy(
                 policy_identifier,
                 name=policy_identifier,
-                policy=json.dumps(policy, sort_keys=True),
+                policy=json_dumps(policy),
                 tags=tags,
             )
             tf_resources.append(policy_tf_resource)
@@ -4299,7 +4316,7 @@ class TerrascriptClient:
         # iam user policy
         values_policy: dict[str, Any] = {
             "name": identifier,
-            "policy": json.dumps(policy, sort_keys=True),
+            "policy": json_dumps(policy),
             "depends_on": self.get_dependencies([user_tf_resource]),
         }
 
@@ -4417,10 +4434,7 @@ class TerrascriptClient:
 
         :return: key is AWS account name and value is terraform configuration
         """
-        return {
-            name: json.dumps(ts, indent=2, sort_keys=True)
-            for name, ts in self.tss.items()
-        }
+        return {name: json_dumps(ts, indent=2) for name, ts in self.tss.items()}
 
     def init_values(
         self, spec: ExternalResourceSpec, init_tags: bool = True
@@ -4532,7 +4546,7 @@ class TerrascriptClient:
             output_name = output_format.format(
                 spec.output_prefix, self.integration_prefix, "annotations"
             )
-            anno_json = json.dumps(spec.annotations()).encode("utf-8")
+            anno_json = json_dumps(spec.annotations()).encode("utf-8")
             output_value = base64.b64encode(anno_json).decode()
             tf_resources.append(Output(output_name, value=output_value))
 
@@ -4672,7 +4686,7 @@ class TerrascriptClient:
         }
         log_groups_policy_values = {
             "policy_name": "es-log-publishing-permissions",
-            "policy_document": json.dumps(log_groups_policy, sort_keys=True),
+            "policy_document": json_dumps(log_groups_policy),
         }
         resource_policy = aws_cloudwatch_log_resource_policy(
             "es_log_publishing_resource_policy",
@@ -5004,7 +5018,7 @@ class TerrascriptClient:
                 }
             ],
         }
-        es_values["access_policies"] = json.dumps(access_policies, sort_keys=True)
+        es_values["access_policies"] = json_dumps(access_policies)
 
         region = values.get("region") or self.default_regions.get(account)
         assert region  # make mypy happy
@@ -5060,7 +5074,7 @@ class TerrascriptClient:
 
                 version_values = {
                     "secret_id": "${" + aws_secret_resource.id + "}",
-                    "secret_string": json.dumps(master_user, sort_keys=True),
+                    "secret_string": json_dumps(master_user),
                 }
                 if provider:
                     version_values["provider"] = provider
@@ -5087,7 +5101,7 @@ class TerrascriptClient:
                 iam_policy_resource = aws_iam_policy(
                     secret_identifier,
                     name=f"{identifier}-secretsmanager-policy",
-                    policy=json.dumps(policy, sort_keys=True),
+                    policy=json_dumps(policy),
                     tags=tags,
                 )
                 tf_resources.append(iam_policy_resource)
@@ -5346,6 +5360,11 @@ class TerrascriptClient:
         )
         if condition_type_key is None:
             raise KeyError(f"unknown alb rule condition type {condition_type}")
+
+        # Query string conditions use a different structure than other condition types
+        if condition_type == "query-string":
+            return {condition_type_key: condition[condition_type_key]}
+
         return {condition_type_key: {"values": condition[condition_type_key]}}
 
     @staticmethod
@@ -5494,7 +5513,7 @@ class TerrascriptClient:
             lb_access_logs_s3_bucket_policy_values = {
                 "provider": provider,
                 "bucket": f"${{{lb_access_logs_s3_bucket_tf_resource.id}}}",
-                "policy": json.dumps(policy, sort_keys=True),
+                "policy": json_dumps(policy),
             }
             lb_access_logs_s3_bucket_policy_tf_resource = aws_s3_bucket_policy(
                 policy_identifier, **lb_access_logs_s3_bucket_policy_values
@@ -5807,9 +5826,13 @@ class TerrascriptClient:
         assert secret  # make mypy happy
         secret_data = self.secret_reader.read_all(secret)
 
+        secret_format = common_values.get("secret_format")
+        if secret_format is not None:
+            secret_data = self._apply_secret_format(str(secret_format), secret_data)
+
         version_values: dict[str, Any] = {
             "secret_id": "${" + aws_secret_resource.id + "}",
-            "secret_string": json.dumps(secret_data, sort_keys=True),
+            "secret_string": json_dumps(secret_data),
         }
 
         if self._multiregion_account(account):
@@ -5830,6 +5853,66 @@ class TerrascriptClient:
 
         self.add_resources(account, tf_resources)
 
+    @staticmethod
+    def _unflatten_dotted_keys_dict(flat_dict: dict[str, str]) -> dict[str, Any]:
+        """Convert a flat dictionary with dotted keys to a nested dictionary.
+
+        Example:
+            {"db.host": "localhost", "db.port": "5432"} ->
+            {"db": {"host": "localhost", "port": "5432"}}
+
+        Raises:
+            ValueError: If there are conflicting keys (e.g., "a.b" and "a.b.c")
+        """
+        result: dict[str, Any] = {}
+        for key, value in flat_dict.items():
+            parts = key.split(".")
+            current = result
+            for i, part in enumerate(parts[:-1]):
+                if part not in current:
+                    current[part] = {}
+                elif not isinstance(current[part], dict):
+                    # Conflict: trying to traverse through a non-dict value
+                    conflicting_path = ".".join(parts[: i + 1])
+                    raise ValueError(
+                        f"Conflicting keys detected: '{conflicting_path}' is both a "
+                        f"value and a nested path in key '{key}'"
+                    )
+                current = current[part]
+
+            # Check if we're trying to set a value where a dict already exists
+            if parts[-1] in current and isinstance(current[parts[-1]], dict):
+                raise ValueError(
+                    f"Conflicting keys detected: '{key}' conflicts with nested keys"
+                )
+
+            current[parts[-1]] = value
+
+        return result
+
+    @staticmethod
+    def _apply_secret_format(
+        secret_format: str, secret_data: dict[str, str]
+    ) -> dict[str, str]:
+        # Convert flat dict with dotted keys to nested dict for Jinja2
+        nested_secret_data = TerrascriptClient._unflatten_dotted_keys_dict(secret_data)
+        rendered_data = process_jinja2_template(secret_format, nested_secret_data)
+
+        parsed_data = json.loads(rendered_data)
+
+        if not isinstance(parsed_data, dict):
+            raise ValueError("secret_format must be a dictionary")
+
+        # validate secret is a dict[str, str]
+        for k, v in parsed_data.items():
+            if not isinstance(k, str):
+                raise ValueError(f"key '{k}' is not a string")
+
+            if not isinstance(v, str):
+                raise ValueError(f"dictionary value '{v}' under '{k}' is not a string")
+
+        return parsed_data
+
     def get_commit_sha(self, repo_info: Mapping) -> str:
         url = repo_info["url"]
         ref = repo_info["ref"]
@@ -5848,7 +5931,8 @@ class TerrascriptClient:
                 return commit.sha
             case "gitlab":
                 gitlab = self.init_gitlab()
-                project = gitlab.get_project(url)
+                if not (project := gitlab.get_project(url)):
+                    raise ValueError(f"could not find gitlab project for url {url}")
                 commits = project.commits.list(ref_name=ref, per_page=1, page=1)
                 return commits[0].id
             case _:
@@ -6129,7 +6213,7 @@ class TerrascriptClient:
         lambda_iam_role_resource = aws_iam_role(
             "lambda_role",
             name=f"ocm-{identifier}-cognito-lambda-role",
-            assume_role_policy=json.dumps(lambda_role_policy),
+            assume_role_policy=json_dumps(lambda_role_policy),
             managed_policy_arns=[lambda_managed_policy_arn],
             force_detach_policies=False,
             max_session_duration=3600,
@@ -6804,7 +6888,7 @@ class TerrascriptClient:
         )
         tf_resources.append(api_gateway_stage_resource)
 
-        rest_api_policy = json.dumps({
+        rest_api_policy = json_dumps({
             "Version": "2012-10-17",
             "Statement": [
                 {
@@ -6908,7 +6992,7 @@ class TerrascriptClient:
                 },
             ],
         }
-        cloudwatch_assume_role_policy = json.dumps(policy, sort_keys=True)
+        cloudwatch_assume_role_policy = json_dumps(policy)
 
         cloudwatch_iam_role_resource = aws_iam_role(
             "cloudwatch_assume_role",
@@ -6936,7 +7020,7 @@ class TerrascriptClient:
             ],
         }
 
-        cloudwatch_iam_policy_document = json.dumps(policy, sort_keys=True)
+        cloudwatch_iam_policy_document = json_dumps(policy)
 
         cloudwatch_iam_policy_resource = aws_iam_policy(
             "cloudwatch",
@@ -7181,7 +7265,7 @@ class TerrascriptClient:
 
                 version_values = {
                     "secret_id": "${" + secret_resource.arn + "}",
-                    "secret_string": json.dumps(secret, sort_keys=True),
+                    "secret_string": json_dumps(secret),
                 }
                 version_resource = aws_secretsmanager_secret_version(
                     secret_identifier, **version_values
@@ -7190,7 +7274,7 @@ class TerrascriptClient:
 
                 secret_policy_values = {
                     "secret_arn": "${" + secret_resource.arn + "}",
-                    "policy": json.dumps({
+                    "policy": json_dumps({
                         "Version": "2012-10-17",
                         "Statement": [
                             {