dbt-platform-helper 15.7.0__tar.gz → 15.9.0__tar.gz

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 15.7.0
+Version: 15.9.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team
@@ -13,27 +13,17 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Jinja2 (==3.1.6)
-Requires-Dist: PyYAML (==6.0.2)
-Requires-Dist: aiohttp (>=3.11.16,<4.0.0)
 Requires-Dist: boto3 (>=1.35.2,<2.0.0)
-Requires-Dist: boto3-stubs (>=1.26.148,<2.0.0)
 Requires-Dist: botocore (>=1.34.85,<2.0.0)
-Requires-Dist: certifi (>=2023.7.22,<2025.0.0)
-Requires-Dist: cfn-flip (==1.3.0)
-Requires-Dist: cfn-lint (>=1.4.2,<2.0.0)
-Requires-Dist: checkov (>=3.2.405,<4.0.0)
+Requires-Dist: cfn-flip (>=1.3.0,<2.0.0)
 Requires-Dist: click (>=8.1.3,<9.0.0)
-Requires-Dist: cryptography (>=44.0.1,<45)
 Requires-Dist: jinja2-simple-tags (>=0.5.0,<0.6.0)
-Requires-Dist: jsonschema (>=4.17.0,<4.18.0)
-Requires-Dist: mypy-boto3-codebuild (>=1.26.0.post1,<2.0.0)
 Requires-Dist: prettytable (>=3.9.0,<4.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: schema (==0.7.5)
-Requires-Dist: semver (>=3.0.2,<4.0.0)
 Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
-Requires-Dist: tomlkit (>=0.12.2,<0.14.0)
 Requires-Dist: yamllint (>=1.35.1,<2.0.0)
 Description-Content-Type: text/markdown
 

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/commands/application.py

@@ -281,12 +281,13 @@ def task_stats(env, app, disk, storage, network):
         memory,
         dsk,
     ) in cpu_response["results"]:
+        cpu_value = f"{float(cpu['value']):.1f}" + "%"
         values = [
             f"{YELLOW}{taskdef['value'].split('-')[-1]}",
             f"{YELLOW}{task['value']}",
             f"{YELLOW}{task_def_revision['value']}",
             f"{YELLOW}{status['value']}",
-            f"{YELLOW}{'%.1f' % float(cpu['value']) + '%'}",
+            f"{YELLOW}{cpu_value}",
             f"{YELLOW}{memory['value'] + 'M'}",
         ]
 

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/constants.py

@@ -16,8 +16,8 @@ TERRAFORM_ECS_SERVICE_MODULE_SOURCE_OVERRIDE_ENV_VAR = (
 TERRAFORM_MODULE_SOURCE_TYPE_ENV_VAR = "TERRAFORM_MODULE_SOURCE_TYPE"  # "LOCAL", "SSH", "OVERRIDE"
 IMAGE_TAG_ENV_VAR = "IMAGE_TAG"
 PLATFORM_HELPER_PACKAGE_NAME = "dbt-platform-helper"
-SUPPORTED_TERRAFORM_VERSION = "~> 1.8"
-SUPPORTED_AWS_PROVIDER_VERSION = "~> 5"
+SUPPORTED_TERRAFORM_VERSION = "~> 1.11"
+SUPPORTED_AWS_PROVIDER_VERSION = "~> 6"
 FIRST_UPGRADABLE_PLATFORM_HELPER_MAJOR_VERSION = 13
 
 MERGED_TPM_PLATFORM_HELPER_VERSION = 14

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/domain/maintenance_page.py

@@ -199,7 +199,13 @@ class MaintenancePage:
                         "AllowedIps",
                         next(rule_priority),
                         service_conditions,
-                        [{"Key": "service", "Value": svc.name}],
+                        [
+                            {"Key": "application", "Value": app},
+                            {"Key": "environment", "Value": env},
+                            {"Key": "reason", "Value": "MaintenancePage"},
+                            {"Key": "managed-by", "Value": "DBT Platform"},
+                            {"Key": "service", "Value": svc.name},
+                        ],
                     )
                     self.load_balancer.create_source_ip_rule(
                         listener_arn,
@@ -208,7 +214,13 @@ class MaintenancePage:
                         "AllowedSourceIps",
                         next(rule_priority),
                         service_conditions,
-                        [{"Key": "service", "Value": svc.name}],
+                        [
+                            {"Key": "application", "Value": app},
+                            {"Key": "environment", "Value": env},
+                            {"Key": "reason", "Value": "MaintenancePage"},
+                            {"Key": "managed-by", "Value": "DBT Platform"},
+                            {"Key": "service", "Value": svc.name},
+                        ],
                     )
 
                 self.load_balancer.create_header_rule(
@@ -219,7 +231,13 @@ class MaintenancePage:
                     "BypassIpFilter",
                     next(rule_priority),
                     service_conditions,
-                    [{"Key": "service", "Value": svc.name}],
+                    [
+                        {"Key": "application", "Value": app},
+                        {"Key": "environment", "Value": env},
+                        {"Key": "reason", "Value": "MaintenancePage"},
+                        {"Key": "managed-by", "Value": "DBT Platform"},
+                        {"Key": "service", "Value": svc.name},
+                    ],
                 )
 
                 # add to accumilating list of conditions for maintenace page rule
@@ -267,8 +285,12 @@ class MaintenancePage:
                         }
                     ],
                     tags=[
+                        {"Key": "application", "Value": app},
+                        {"Key": "environment", "Value": env},
+                        {"Key": "reason", "Value": "MaintenancePage"},
                         {"Key": "name", "Value": "MaintenancePage"},
                         {"Key": "type", "Value": template},
+                        {"Key": "managed-by", "Value": "DBT Platform"},
                     ],
                 )
         except Exception as e:

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/domain/pipelines.py

@@ -54,6 +54,25 @@ class Pipelines:
             or self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
         )
 
+    def _map_environment_pipeline_accounts(self, platform_config) -> list[tuple[str, str]]:
+        environment_pipelines_config = platform_config[ENVIRONMENT_PIPELINES_KEY]
+        environment_config = platform_config["environments"]
+
+        account_id_lookup = {
+            env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
+            for env in environment_config.values()
+            if env is not None and "accounts" in env and "deploy" in env["accounts"]
+        }
+
+        accounts = set()
+
+        for config in environment_pipelines_config.values():
+            account = config.get("account")
+            deploy_account_id = account_id_lookup.get(account)
+            accounts.add((account, deploy_account_id))
+
+        return list(accounts)
+
     def generate(
         self,
         deploy_branch: str,
@@ -107,20 +126,16 @@ class Pipelines:
         )
 
         if has_environment_pipelines:
-            environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
-            accounts = {
-                config.get("account")
-                for config in environment_pipelines.values()
-                if "account" in config
-            }
+            accounts = self._map_environment_pipeline_accounts(platform_config)
 
-            for account in accounts:
+            for account_name, account_id in accounts:
                 self._generate_terraform_environment_pipeline_manifest(
                     platform_config["application"],
                     deploy_repository,
-                    account,
+                    account_name,
                     env_pipeline_module_source,
                     deploy_branch,
+                    account_id,
                 )
 
         if has_codebase_pipelines:
@@ -163,6 +178,7 @@ class Pipelines:
         aws_account: str,
         module_source: str,
         deploy_branch: str,
+        aws_account_id: str,
     ):
         env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
 
@@ -175,6 +191,7 @@ class Pipelines:
                 "deploy_branch": deploy_branch,
                 "terraform_version": SUPPORTED_TERRAFORM_VERSION,
                 "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,
+                "deploy_account_id": aws_account_id,
             }
         )
 

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/entities/platform_config_schema.py

@@ -410,11 +410,19 @@ class PlatformConfigSchema:
             Optional("environments"): {
                 Optional(PlatformConfigSchema.__valid_environment_name()): {
                     "team_name": str,
-                    "contact_name": str,
-                    "contact_email": str,
-                    "documentation_url": str,
+                    Optional("contact_name"): str,
+                    Optional("contact_email"): str,
+                    Optional("contacts"): [
+                        {
+                            "name": str,
+                            "type": str,
+                            "contact": str,
+                        }
+                    ],
+                    Optional("documentation_url"): str,
                     "services_to_monitor": dict,
-                }
+                    Optional("description"): str,
+                },
             },
         }
 

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/entities/service.py

@@ -54,7 +54,7 @@ class SidecarOverride(BaseModel):
 class Image(BaseModel):
     build: Optional[str] = Field(default=None)
     location: Optional[str] = Field(default=None)
-    port: int = Field()
+    port: Optional[int] = Field(default=None)
 
 
 class VPC(BaseModel):

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/providers/load_balancers.py

@@ -149,8 +149,9 @@ class LoadBalancerProvider:
 
         for rule in rules:
             for action in rule["Actions"]:
-                if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
-                    conditions = rule["Conditions"]
+                if "TargetGroupArn" in action:
+                    if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                        conditions = rule["Conditions"]
 
         if not conditions:
             raise ListenerRuleConditionsNotFoundException(listener_arn)

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/providers/terraform_manifest.py

@@ -30,6 +30,7 @@ class TerraformManifestProvider:
         service_dir = f"terraform/services/{environment}/{config_object.name}"
         platform_config = ConfigProvider.apply_environment_defaults(platform_config)
         account = self._get_account_for_env(environment, platform_config)
+        deploy_to_account_id = self._get_account_id_for_account(account, platform_config)
         state_key_suffix = f"{config_object.name}-{environment}"
 
         terraform = {}
@@ -37,7 +38,7 @@ class TerraformManifestProvider:
 
         self._add_service_locals(terraform, environment, image_tag)
 
-        self._add_provider(terraform, account)
+        self._add_provider(terraform, account, deploy_to_account_id)
         self._add_backend(
             terraform, platform_config, account, f"tfstate/services/{state_key_suffix}.tfstate"
         )
@@ -54,11 +55,13 @@ class TerraformManifestProvider:
             "application": '${local.platform_config["application"]}',
             "environments": '${local.platform_config["environments"]}',
             "env_config": '${{for name, config in local.environments: name => merge(lookup(local.environments, "*", {}), config)}}',
-            "service_config": '${yamldecode(templatefile("./service-config.yml", {COPILOT_ENVIRONMENT_NAME = local.environment, IMAGE_TAG = local.image_tag}))}',
+            "service_config": '${yamldecode(templatefile("./service-config.yml", {PLATFORM_ENVIRONMENT_NAME = local.environment, IMAGE_TAG = local.image_tag}))}',
             "raw_env_config": '${local.platform_config["environments"]}',
             "combined_env_config": '${{for name, config in local.raw_env_config: name => merge(lookup(local.raw_env_config, "*", {}), config)}}',
             "service_deployment_mode": '${lookup(local.combined_env_config[local.environment], "service-deployment-mode", "copilot")}',
             "non_copilot_service_deployment_mode": '${local.service_deployment_mode == "dual-deploy-copilot-traffic" || local.service_deployment_mode == "dual-deploy-platform-traffic" || local.service_deployment_mode == "platform" ? 1 : 0}',
+            "custom_iam_policy_path": '${abspath(format("%s/../../../../services/%s/custom-iam-policy/%s.yml", path.module, local.service_config.name, local.environment))}',
+            "custom_iam_policy_json": "${fileexists(local.custom_iam_policy_path) ? jsonencode(yamldecode(file(local.custom_iam_policy_path))) : null}",
         }
 
     def _add_service_module(
@@ -76,6 +79,8 @@ class TerraformManifestProvider:
                 "environment": "${local.environment}",
                 "service_config": "${local.service_config}",
                 "env_config": "${local.env_config}",
+                "platform_extensions": '${local.platform_config["extensions"]}',
+                "custom_iam_policy_json": "${local.custom_iam_policy_json}",
             }
         }
 
@@ -88,12 +93,13 @@ class TerraformManifestProvider:
         module_source: str,
     ):
         default_account = self._get_account_for_env("*", platform_config)
+        deploy_to_account_id = self._get_account_id_for_account(default_account, platform_config)
         state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
 
         terraform = {}
         self._add_header(terraform)
         self._add_codebase_pipeline_locals(terraform)
-        self._add_provider(terraform, default_account)
+        self._add_provider(terraform, default_account, deploy_to_account_id)
         self._add_backend(
             terraform,
             platform_config,
@@ -142,6 +148,16 @@ class TerraformManifestProvider:
         )
         return account
 
+    @staticmethod
+    def _get_account_id_for_account(account_name, platform_config):
+        environment_config = platform_config["environments"]
+        account_id_lookup = {
+            env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
+            for env in environment_config.values()
+            if env is not None and "accounts" in env and "deploy" in env["accounts"]
+        }
+        return account_id_lookup.get(account_name)
+
     @staticmethod
     def _add_header(terraform: dict):
         time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -160,12 +176,11 @@ class TerraformManifestProvider:
         }
 
     @staticmethod
-    def _add_provider(terraform: dict, default_account: str):
+    def _add_provider(terraform: dict, deploy_to_account: str, deploy_to_account_id: str):
         terraform["provider"] = {"aws": {}}
         terraform["provider"]["aws"]["region"] = "eu-west-2"
-        terraform["provider"]["aws"]["profile"] = default_account
-        terraform["provider"]["aws"]["alias"] = default_account
-        terraform["provider"]["aws"]["shared_credentials_files"] = ["~/.aws/config"]
+        terraform["provider"]["aws"]["profile"] = deploy_to_account
+        terraform["provider"]["aws"]["allowed_account_ids"] = [deploy_to_account_id]
 
     @staticmethod
     def _add_backend(terraform: dict, platform_config: dict, account: str, state_key: str):

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/providers/yaml_file.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from pathlib import Path
 
 import yaml
@@ -84,6 +85,32 @@ class YamlFileProvider:
         if duplicate_keys:
             raise DuplicateKeysException(",".join(duplicate_keys))
 
+    @staticmethod
+    def remove_empty_keys(config: (dict, OrderedDict)) -> (dict, OrderedDict):
+        cleaned = config.__class__()
+
+        for k, v in config.items():
+            if isinstance(v, (dict, OrderedDict)):
+                v = YamlFileProvider.remove_empty_keys(v)
+            if v not in ("", None, [], {}, ()):
+                cleaned[k] = v
+
+        return cleaned
+
+    @staticmethod
+    def find_and_replace(config, string: str, replacement: str):
+        if isinstance(config, (dict, OrderedDict)):
+            return {
+                k: YamlFileProvider.find_and_replace(v, string, replacement)
+                for k, v in config.items()
+            }
+        elif isinstance(config, list):
+            return [YamlFileProvider.find_and_replace(item, string, replacement) for item in config]
+        elif isinstance(config, str):
+            return config.replace(string, replacement)
+        else:
+            return replacement if config == string else config
+
 
 def account_number_representer(dumper, data):
     if data.isdigit():

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -10,10 +10,10 @@ locals {
 provider "aws" {
   region                   = "eu-west-2"
   profile                  = "{{ aws_account }}"
-  alias                    = "{{ aws_account }}"
-  shared_credentials_files = ["~/.aws/config"]
+  allowed_account_ids      = ["{{ deploy_account_id }}"]
 }
 
+
 terraform {
   required_version = "{{ terraform_version }}"
   backend "s3" {

{dbt_platform_helper-15.7.0 → dbt_platform_helper-15.9.0}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "15.7.0"
+version = "15.9.0"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"
@@ -21,41 +21,36 @@ platform-helper = "platform_helper:platform_helper"
 
 [tool.poetry.dependencies]
 Jinja2 = "3.1.6"
-PyYAML = "6.0.2"
 yamllint = "^1.35.1"
 boto3 = "^1.35.2"
-boto3-stubs = "^1.26.148"
 botocore = "^1.34.85"
 click = "^8.1.3"
-mypy-boto3-codebuild = "^1.26.0.post1"
 python = ">3.9.1,<4.0"
 schema = "0.7.5"
-cfn-flip = "1.3.0"
-aiohttp = "^3.11.16"
-certifi = ">=2023.7.22,<2025.0.0"
-cryptography = ">=44.0.1, <45"
 jinja2-simple-tags = "^0.5.0"
 requests = "^2.31.0"
 prettytable = "^3.9.0"
-semver = "^3.0.2"
-tomlkit = ">=0.12.2,<0.14.0"
 slack-sdk = "^3.27.1"
-jsonschema = "~=4.17.0"
-cfn-lint = "^1.4.2"
 psycopg2-binary = "^2.9.9"
-checkov = "^3.2.405"
+pydantic = "^2.11.7"
+cfn-flip = "^1.3.0"
 
 [tool.poetry.group.dev.dependencies]
+checkov = "^3.2.405"
 moto = {extras = ["all"], version = "^5.0.28"}
 pyfakefs = "^5.2.2"
 pytest = "^8.3.5"
 pytest-env = ">=0.8.1,<1.2.0"
 tox = "^4.6.0"
-pre-commit = "^3.3.3"
+pre-commit = ">=3.3.3,<5.0.0"
 freezegun = "^1.2.2"
 parameterized = "^0.9.0"
 pytest-xdist = "^3.5.0"
 pytest-cov = "^6.0.0"
+mypy-boto3-codebuild = "^1.40.8"
+
+python-hcl2 = "^7.3.1"
+boto3-stubs =  "^1.26.148"
 
 [build-system]
 requires = ["poetry-core"]