dbt-platform-helper 15.3.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/providers/s3.py

@@ -0,0 +1,21 @@
+import boto3
+from botocore.exceptions import ClientError
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class S3Provider:
+
+    def __init__(self, client: boto3.client):
+        self.client = client
+
+    def get_object(self, bucket_name: str, object_key: str) -> str:
+        """Returns an object from an S3 bucket."""
+
+        try:
+            content = self.client.get_object(Bucket=bucket_name, Key=object_key)
+            return content["Body"].read().decode("utf-8")
+        except ClientError as e:
+            raise PlatformException(
+                f"Failed to get '{object_key}' from '{bucket_name}'. Error: {e}"
+            )

dbt_platform_helper/providers/terraform_manifest.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from importlib.metadata import version
 from pathlib import Path
 
+from dbt_platform_helper.constants import EXTENSIONS_MODULE_PATH
 from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
 from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
 from dbt_platform_helper.providers.config import ConfigProvider
@@ -17,6 +18,74 @@ class TerraformManifestProvider:
         self.file_provider = file_provider
         self.io = io
 
+    def generate_service_config(
+        self,
+        config_object,
+        environment,
+        platform_helper_version: str,
+        platform_config,
+        module_source_override: str = None,
+    ):
+
+        service_dir = f"terraform/services/{environment}/{config_object.name}"
+        platform_config = ConfigProvider.apply_environment_defaults(platform_config)
+        account = self._get_account_for_env(environment, platform_config)
+        deploy_to_account_id = self._get_account_id_for_account(account, platform_config)
+        application_name = platform_config["application"]
+
+        terraform = {}
+        self._add_header(terraform)
+
+        self._add_service_locals(terraform, environment)
+
+        self._add_provider(terraform, account, deploy_to_account_id)
+        self._add_backend(
+            terraform,
+            platform_config,
+            account,
+            f"tfstate/application/{application_name}/services/{environment}/{config_object.name}.tfstate",
+        )
+
+        self._add_service_module(terraform, platform_helper_version, module_source_override)
+
+        self._write_terraform_json(terraform, service_dir)
+
+    def _add_service_locals(self, terraform, environment):
+        terraform["locals"] = {
+            "environment": environment,
+            "platform_config": '${yamldecode(file("../../../../platform-config.yml"))}',
+            "application": '${local.platform_config["application"]}',
+            "environments": '${local.platform_config["environments"]}',
+            "env_config": '${{for name, config in local.environments: name => merge(lookup(local.environments, "*", {}), config)}}',
+            "service_config": '${yamldecode(file("./service-config.yml"))}',
+            "raw_env_config": '${local.platform_config["environments"]}',
+            "combined_env_config": '${{for name, config in local.raw_env_config: name => merge(lookup(local.raw_env_config, "*", {}), config)}}',
+            "service_deployment_mode": '${lookup(local.combined_env_config[local.environment], "service-deployment-mode", "copilot")}',
+            "non_copilot_service_deployment_mode": '${local.service_deployment_mode == "dual-deploy-copilot-traffic" || local.service_deployment_mode == "dual-deploy-platform-traffic" || local.service_deployment_mode == "platform" ? 1 : 0}',
+            "custom_iam_policy_path": '${abspath(format("%s/../../../../services/%s/custom-iam-policy/%s.yml", path.module, local.service_config.name, local.environment))}',
+            "custom_iam_policy_json": "${fileexists(local.custom_iam_policy_path) ? jsonencode(yamldecode(file(local.custom_iam_policy_path))) : null}",
+        }
+
+    def _add_service_module(
+        self, terraform: dict, platform_helper_version: str, module_source_override: str = None
+    ):
+        source = (
+            module_source_override
+            or f"git::git@github.com:uktrade/platform-tools.git//terraform/ecs-service?depth=1&ref={platform_helper_version}"
+        )
+        terraform["module"] = {
+            "ecs-service": {
+                "source": source,
+                "count": "${local.non_copilot_service_deployment_mode}",
+                "application": "${local.application}",
+                "environment": "${local.environment}",
+                "service_config": "${local.service_config}",
+                "env_config": "${local.env_config}",
+                "platform_extensions": '${local.platform_config["extensions"]}',
+                "custom_iam_policy_json": "${local.custom_iam_policy_json}",
+            }
+        }
+
     def generate_codebase_pipeline_config(
         self,
         platform_config: dict,
@@ -26,13 +95,19 @@ class TerraformManifestProvider:
         module_source: str,
     ):
         default_account = self._get_account_for_env("*", platform_config)
+        deploy_to_account_id = self._get_account_id_for_account(default_account, platform_config)
         state_key_suffix = f"{platform_config['application']}-codebase-pipelines"
 
         terraform = {}
         self._add_header(terraform)
         self._add_codebase_pipeline_locals(terraform)
-        self._add_provider(terraform, default_account)
-        self._add_backend(terraform, platform_config, default_account, state_key_suffix)
+        self._add_provider(terraform, default_account, deploy_to_account_id)
+        self._add_backend(
+            terraform,
+            platform_config,
+            default_account,
+            f"tfstate/application/{state_key_suffix}.tfstate",
+        )
         self._add_codebase_pipeline_module(
             terraform, platform_helper_version, deploy_repository, module_source
         )
@@ -56,7 +131,9 @@ class TerraformManifestProvider:
         terraform = {}
         self._add_header(terraform)
         self._add_environment_locals(terraform, application_name)
-        self._add_backend(terraform, platform_config, account, state_key_suffix)
+        self._add_backend(
+            terraform, platform_config, account, f"tfstate/application/{state_key_suffix}.tfstate"
+        )
         self._add_extensions_module(terraform, platform_helper_version, env, module_source_override)
         self._add_moved(terraform, platform_config)
         self._ensure_no_hcl_manifest_file(env_dir)
@@ -73,6 +150,16 @@ class TerraformManifestProvider:
         )
         return account
 
+    @staticmethod
+    def _get_account_id_for_account(account_name, platform_config):
+        environment_config = platform_config["environments"]
+        account_id_lookup = {
+            env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
+            for env in environment_config.values()
+            if env is not None and "accounts" in env and "deploy" in env["accounts"]
+        }
+        return account_id_lookup.get(account_name)
+
     @staticmethod
     def _add_header(terraform: dict):
         time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -91,21 +178,20 @@ class TerraformManifestProvider:
         }
 
     @staticmethod
-    def _add_provider(terraform: dict, default_account: str):
+    def _add_provider(terraform: dict, deploy_to_account: str, deploy_to_account_id: str):
         terraform["provider"] = {"aws": {}}
         terraform["provider"]["aws"]["region"] = "eu-west-2"
-        terraform["provider"]["aws"]["profile"] = default_account
-        terraform["provider"]["aws"]["alias"] = default_account
-        terraform["provider"]["aws"]["shared_credentials_files"] = ["~/.aws/config"]
+        terraform["provider"]["aws"]["profile"] = deploy_to_account
+        terraform["provider"]["aws"]["allowed_account_ids"] = [deploy_to_account_id]
 
     @staticmethod
-    def _add_backend(terraform: dict, platform_config: dict, account: str, state_key_suffix: str):
+    def _add_backend(terraform: dict, platform_config: dict, account: str, state_key: str):
         terraform["terraform"] = {
             "required_version": SUPPORTED_TERRAFORM_VERSION,
             "backend": {
                 "s3": {
                     "bucket": f"terraform-platform-state-{account}",
-                    "key": f"tfstate/application/{state_key_suffix}.tfstate",
+                    "key": state_key,
                     "region": "eu-west-2",
                     "encrypt": True,
                     "kms_key_id": f"alias/terraform-platform-state-s3-key-{account}",
@@ -135,6 +221,7 @@ class TerraformManifestProvider:
                 "deploy_repository": f"{deploy_repository}",
                 "deploy_repository_branch": '${lookup(each.value, "deploy_repository_branch", "main")}',
                 "additional_ecr_repository": '${lookup(each.value, "additional_ecr_repository", null)}',
+                "cache_invalidation": '${lookup(each.value, "cache_invalidation", null)}',
                 "pipelines": '${lookup(each.value, "pipelines", [])}',
                 "services": "${each.value.services}",
                 "requires_image_build": '${lookup(each.value, "requires_image_build", true)}',
@@ -148,10 +235,7 @@ class TerraformManifestProvider:
     def _add_extensions_module(
         terraform: dict, platform_helper_version: str, env: str, module_source_override: str = None
     ):
-        source = (
-            module_source_override
-            or f"git::git@github.com:uktrade/platform-tools.git//terraform/extensions?depth=1&ref={platform_helper_version}"
-        )
+        source = module_source_override or f"{EXTENSIONS_MODULE_PATH}{platform_helper_version}"
         terraform["module"] = {
             "extensions": {
                 "source": source,

dbt_platform_helper/providers/vpc.py

@@ -78,11 +78,42 @@ class VpcProvider:
 
     def _get_security_groups(self, app: str, env: str, vpc_id: str) -> list:
         vpc_filter = {"Name": "vpc-id", "Values": [vpc_id]}
-        # TODO Handle terraformed environment SG https://uktrade.atlassian.net/browse/DBTP-2074
-        tag_filter = {"Name": f"tag:Name", "Values": [f"copilot-{app}-{env}-env"]}
-        response = self.ec2_client.describe_security_groups(Filters=[vpc_filter, tag_filter])
-
-        return [sg.get("GroupId") for sg in response.get("SecurityGroups")]
+        platform_sg_name = f"platform-{app}-{env}-env-sg"
+        copilot_sg_name = f"copilot-{app}-{env}-env"
+        tag_filter = {"Name": f"tag:Name", "Values": [copilot_sg_name, platform_sg_name]}
+
+        filtered_security_groups = self.ec2_client.describe_security_groups(
+            Filters=[vpc_filter, tag_filter]
+        )
+
+        platform_security_groups = self._get_matching_security_groups(
+            filtered_security_groups, platform_sg_name
+        )
+
+        if platform_security_groups:
+            print(
+                f"using {platform_security_groups}"
+            )  # TODO remove this once decopilotiing has been completed
+            return platform_security_groups
+
+        copilot_security_groups = self._get_matching_security_groups(
+            filtered_security_groups, copilot_sg_name
+        )
+
+        print(
+            f"using {copilot_security_groups}"
+        )  # TODO remove this once decopilotiing has been completed
+        return copilot_security_groups
+
+    def _get_matching_security_groups(
+        self, filtered_security_groups: list[dict], security_group_name: str
+    ):
+        matching_sec_groups = filtered_security_groups.get("SecurityGroups")
+        return [
+            sg.get("GroupId")
+            for sg in matching_sec_groups
+            if {"Key": "Name", "Value": security_group_name} in sg.get("Tags", [])
+        ]
 
     def get_vpc(self, app: str, env: str, vpc_name: str) -> Vpc:
 

dbt_platform_helper/providers/yaml_file.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from pathlib import Path
 
 import yaml
@@ -84,6 +85,40 @@ class YamlFileProvider:
         if duplicate_keys:
             raise DuplicateKeysException(",".join(duplicate_keys))
 
+    @staticmethod
+    def remove_empty_keys(config: (dict, OrderedDict)) -> (dict, OrderedDict):
+        cleaned = config.__class__()
+
+        for k, v in config.items():
+            if isinstance(v, (dict, OrderedDict)):
+                v = YamlFileProvider.remove_empty_keys(v)
+            if v not in (None, [], {}, ()):
+                cleaned[k] = v
+
+        return cleaned
+
+    @staticmethod
+    def find_and_replace(config, strings: list, replacements: list):
+        if len(strings) != len(replacements):
+            raise ValueError("'strings' and 'replacements' must be the same length.")
+        if not isinstance(strings, list) or not isinstance(replacements, list):
+            raise ValueError("'strings' and 'replacements' must both be lists.")
+        if isinstance(config, (dict, OrderedDict)):
+            return {
+                k: YamlFileProvider.find_and_replace(v, strings, replacements)
+                for k, v in config.items()
+            }
+        elif isinstance(config, list):
+            return [
+                YamlFileProvider.find_and_replace(item, strings, replacements) for item in config
+            ]
+        elif isinstance(config, str):
+            for s, r in zip(strings, replacements):
+                config = config.replace(s, r)
+            return config
+        else:
+            return replacements if config == strings else config
+
 
 def account_number_representer(dumper, data):
     if data.isdigit():

dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -10,10 +10,10 @@ locals {
 provider "aws" {
   region                   = "eu-west-2"
   profile                  = "{{ aws_account }}"
-  alias                    = "{{ aws_account }}"
-  shared_credentials_files = ["~/.aws/config"]
+  allowed_account_ids      = ["{{ deploy_account_id }}"]
 }
 
+
 terraform {
   required_version = "{{ terraform_version }}"
   backend "s3" {
@@ -49,4 +49,5 @@ module "environment-pipelines" {
   slack_channel       = each.value.slack_channel
   trigger_on_push     = each.value.trigger_on_push
   pipeline_to_trigger = lookup(each.value, "pipeline_to_trigger", null)
+  pinned_version      = {% if pinned_version  %}"{{ pinned_version }}"{% else %}null{% endif %}
 }

dbt_platform_helper/templates/svc/overrides/cfn.patches.yml

@@ -24,3 +24,8 @@
     Condition:
       StringEquals:
           'ssm:ResourceTag/copilot-application': '__all__'
+
+- op: add
+  path: /Resources/TaskDefinition/Properties/pidMode
+  value:
+    task

dbt_platform_helper/utils/application.py

@@ -59,23 +59,11 @@ class Application:
         return str(self) == str(other)
 
 
-def load_application(app=None, default_session=None) -> Application:
+def load_application(app=None, default_session=None, env=None) -> Application:
     application = Application(app if app else get_application_name())
     current_session = default_session if default_session else get_aws_session_or_abort()
 
     ssm_client = current_session.client("ssm")
-
-    try:
-        ssm_client.get_parameter(
-            Name=f"/copilot/applications/{application.name}",
-            WithDecryption=False,
-        )
-    except ssm_client.exceptions.ParameterNotFound:
-        raise ApplicationNotFoundException(application.name)
-
-    path = f"/copilot/applications/{application.name}/environments"
-    secrets = get_ssm_secrets(app, None, current_session, path)
-
     sts_client = current_session.client("sts")
     account_id = sts_client.get_caller_identity()["Account"]
     sessions = {account_id: current_session}
@@ -86,20 +74,112 @@ def load_application(app=None, default_session=None) -> Application:
         nesting.
 
         e.g.
+         - /platform/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env will match.
          - /copilot/applications/test/environments/my_env/addons will not match.
         """
-        environment_key_regex = r"^/copilot/applications/{}/environments/[^/]*$".format(
+        environment_key_regex = r"^/(copilot|platform)/applications/{}/environments/[^/]*$".format(
             application.name
         )
         return bool(re.match(environment_key_regex, name))
 
-    environments = {
+    environments_data = []
+
+    # Try to load all /platform SSM parameters that are present
+    env_params = get_ssm_secrets(
+        app=app,
+        env=None,
+        session=current_session,
+        path=f"/platform/applications/{application.name}/environments",
+    )
+
+    if env_params:
+        for name, value in env_params:
+            try:
+                param_data = json.loads(value)
+            except json.JSONDecodeError:
+                continue
+
+            # Each /platform SSM parameter contains data about all the environments of an application
+            if "allEnvironments" in param_data:
+                environments_data = param_data["allEnvironments"]
+                break  # Only need one
+    else:
+        try:
+            # Check that the Copilot application exists
+            ssm_client.get_parameter(
+                Name=f"/copilot/applications/{application.name}",
+                WithDecryption=False,
+            )
+
+            # Legacy /copilot SSM parameters for each environment
+            env_params = get_ssm_secrets(
+                app, None, current_session, f"/copilot/applications/{application.name}/environments"
+            )
+
+            for name, value in env_params:
+                try:
+                    param_data = json.loads(value)
+                except json.JSONDecodeError:
+                    continue
+
+                if is_environment_key(name):
+                    environments_data.append(param_data)
+
+        except ssm_client.exceptions.ParameterNotFound:
+            raise ApplicationNotFoundException(
+                application_name=application.name, environment_name=env
+            )
+
+    application.environments = {
         env["name"]: Environment(env["name"], env["accountID"], sessions)
-        for env in [json.loads(s[1]) for s in secrets if is_environment_key(s[0])]
+        for env in environments_data
     }
-    application.environments = environments
 
+    application.services = _load_services(ssm_client, application)
+
+    return application
+
+
+def _load_services(ssm_client, application: Application) -> Dict[str, Service]:
+    """
+    Try to load
+    /platform/applications/{app}/environments/{env}/services/{service}
+    parameters if present.
+
+    Otherwise, fall back to legacy /copilot/applications/{app}/components
+    parameters.
+    """
+    services: Dict[str, Service] = {}
+
+    # Try /platform SSM parameter
+    for env_name in application.environments.keys():
+        params = dict(
+            Path=f"/platform/applications/{application.name}/environments/{env_name}/services",
+            Recursive=False,
+            WithDecryption=False,
+        )
+
+        while True:
+            response = ssm_client.get_parameters_by_path(**params)
+            for ssm_param in response.get("Parameters", []):
+                try:
+                    data = json.loads(ssm_param["Value"])
+                    name = data["name"]
+                    kind = data["type"]
+                    services.setdefault(name, Service(name, kind))  # Avoid duplicates
+                except (json.JSONDecodeError, KeyError):
+                    continue
+
+            if "NextToken" in response:
+                params["NextToken"] = response["NextToken"]
+            else:
+                break
+
+    if services:
+        return services
+
+    # Fallback to legacy /copilot SSM parameter
     response = ssm_client.get_parameters_by_path(
         Path=f"/copilot/applications/{application.name}/components",
         Recursive=False,
@@ -115,12 +195,12 @@ def load_application(app=None, default_session=None) -> Application:
         )
         results.extend(response["Parameters"])
 
-    application.services = {
+    legacy_services = {
         svc["name"]: Service(svc["name"], svc["type"])
         for svc in [json.loads(parameter["Value"]) for parameter in results]
     }
 
-    return application
+    return legacy_services
 
 
 def get_application_name(abort=abort_with_error):
@@ -142,9 +222,12 @@ class ApplicationException(PlatformException):
 
 
 class ApplicationNotFoundException(ApplicationException):
-    def __init__(self, application_name: str):
+    def __init__(self, application_name: str, environment_name: str):
         super().__init__(
-            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}"; ensure you have set the environment variable "AWS_PROFILE" correctly."""
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}". 
+Please ensure that the environment variable "AWS_PROFILE" is set correctly. If the issue persists, verify that one of the following AWS SSM parameters exists:
+ - /platform/applications/{application_name}/environments/{environment_name}
+ - /copilot/applications/{application_name}"""
         )
 
 

dbt_platform_helper/utils/aws.py

@@ -158,7 +158,17 @@ def get_ssm_secrets(app, env, session=None, path=None):
     secrets = []
 
     while True:
-        response = client.get_parameters_by_path(**params)
+        try:
+            response = client.get_parameters_by_path(**params)
+        except ClientError as e:
+            if e.response["Error"]["Code"] == "AccessDeniedException":
+                click.secho(
+                    "Access denied on SSM, due to missing permissions. Please update your environment infrastructure.",
+                    fg="magenta",
+                )
+                break
+            else:
+                raise e
 
         for secret in response["Parameters"]:
             secrets.append((secret["Name"], secret["Value"]))
@@ -202,15 +212,6 @@ def set_ssm_param(
     client.put_parameter(**parameter_args)
 
 
-def get_codestar_connection_arn(app_name):
-    session = get_aws_session_or_abort()
-    response = session.client("codestar-connections").list_connections()
-
-    for connection in response["Connections"]:
-        if connection["ConnectionName"] == app_name:
-            return connection["ConnectionArn"]
-
-
 def get_account_details(sts_client=None):
     if not sts_client:
         sts_client = get_aws_session_or_abort().client("sts")

dbt_platform_helper/utils/deep_merge.py

@@ -0,0 +1,10 @@
+def deep_merge(base, override):
+    result = base.copy()
+
+    for k, v in override.items():
+        if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+            result[k] = deep_merge(result[k], v)
+        else:
+            result[k] = v
+
+    return result

dbt_platform_helper/utils/git.py

@@ -13,4 +13,4 @@ def extract_repository_name(repository_url):
     if not repository_url:
         return
 
-    return re.search(r"([^/:]*/[^/]*)\.git", repository_url).group(1)
+    return re.search(r"([^/:]*/[^/]*?)(?:\.git)?$", repository_url).group(1)

{dbt_platform_helper-15.3.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA

@@ -1,40 +1,31 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: dbt-platform-helper
-Version: 15.3.0
+Version: 15.16.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
+License-File: LICENSE
 Author: Department for Business and Trade Platform Team
 Author-email: sre-team@digital.trade.gov.uk
-Requires-Python: >=3.9, !=2.7.*, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*, !=3.7.*, !=3.8.*
+Requires-Python: >3.9.1,<4.0
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Requires-Dist: Jinja2 (==3.1.6)
-Requires-Dist: PyYAML (==6.0.1)
-Requires-Dist: aiohttp (>=3.11.16,<4.0.0)
 Requires-Dist: boto3 (>=1.35.2,<2.0.0)
-Requires-Dist: boto3-stubs (>=1.26.148,<2.0.0)
 Requires-Dist: botocore (>=1.34.85,<2.0.0)
-Requires-Dist: certifi (>=2023.7.22,<2025.0.0)
-Requires-Dist: cfn-flip (==1.3.0)
-Requires-Dist: cfn-lint (>=1.4.2,<2.0.0)
-Requires-Dist: checkov (>=3.2.405,<4.0.0)
+Requires-Dist: cfn-flip (>=1.3.0,<2.0.0)
 Requires-Dist: click (>=8.1.3,<9.0.0)
-Requires-Dist: cloudfoundry-client (==1.35.2)
-Requires-Dist: cryptography (>=44.0.1,<45)
-Requires-Dist: jinja2-simple-tags (>=0.5.0,<0.6.0)
-Requires-Dist: jsonschema (>=4.17.0,<4.18.0)
-Requires-Dist: mypy-boto3-codebuild (>=1.26.0.post1,<2.0.0)
+Requires-Dist: jinja2-simple-tags (>=0.5,<0.7)
 Requires-Dist: prettytable (>=3.9.0,<4.0.0)
 Requires-Dist: psycopg2-binary (>=2.9.9,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: schema (==0.7.5)
-Requires-Dist: semver (>=3.0.2,<4.0.0)
 Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
-Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
 Requires-Dist: yamllint (>=1.35.1,<2.0.0)
 Description-Content-Type: text/markdown