dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.1__py3-none-any.whl

dbt_platform_helper/domain/maintenance_page.py

@@ -3,38 +3,62 @@ import random
 import re
 import string
 from pathlib import Path
+from typing import Callable
 from typing import List
 from typing import Union
 
 import boto3
 import click
 
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
 from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
 from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
 from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import Environment
 from dbt_platform_helper.utils.application import Service
 from dbt_platform_helper.utils.application import load_application
 
 
-class MaintenancePageProvider:
-    def activate(self, app, env, svc, template, vpc):
-        application = load_application(app)
-        application_environment = get_app_environment(app, env)
+class MaintenancePageException(PlatformException):
+    pass
+
+
+class LoadBalancedWebServiceNotFoundException(MaintenancePageException):
+    pass
+
+
+class MaintenancePage:
+    def __init__(
+        self,
+        user_prompt_callback: Callable[[str], bool] = click.confirm,
+        echo: Callable[[str], str] = click.secho,
+    ):
+        self.user_prompt_callback = user_prompt_callback
+        self.echo = echo
 
+    def _get_deployed_load_balanced_web_services(self, app: Application, svc: str):
         if "*" in svc:
-            services = [
-                s for s in application.services.values() if s.kind == "Load Balanced Web Service"
-            ]
+            services = [s for s in app.services.values() if s.kind == "Load Balanced Web Service"]
         else:
-            all_services = [get_app_service(app, s) for s in list(svc)]
+            all_services = [get_app_service(app.name, s) for s in list(svc)]
             services = [s for s in all_services if s.kind == "Load Balanced Web Service"]
-
         if not services:
-            click.secho(f"No services deployed yet to {app} environment {env}", fg="red")
+            raise LoadBalancedWebServiceNotFoundException
+        return services
+
+    def activate(self, app, env, svc, template, vpc):
+        try:
+            services = self._get_deployed_load_balanced_web_services(load_application(app), svc)
+        except LoadBalancedWebServiceNotFoundException:
+            # TODO DBTP-1643 - this bit of logic does not depend on env, so env shouldn't really be in the exception
+            # message
+            # Exception should be propagated to command and caught there.
+            self.echo(f"No services deployed yet to {app} environment {env}", fg="red")
             raise click.Abort
 
+        application_environment = get_app_environment(app, env)
         try:
             https_listener = find_https_listener(application_environment.session, app, env)
             current_maintenance_page = get_maintenance_page(
@@ -42,7 +66,7 @@ class MaintenancePageProvider:
             )
             remove_current_maintenance_page = False
             if current_maintenance_page:
-                remove_current_maintenance_page = click.confirm(
+                remove_current_maintenance_page = self.user_prompt_callback(
                     f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
                     f"environment in {app}.\nWould you like to replace it with a '{template}' "
                     f"maintenance page?"
@@ -50,7 +74,7 @@ class MaintenancePageProvider:
                 if not remove_current_maintenance_page:
                     raise click.Abort
 
-            if remove_current_maintenance_page or click.confirm(
+            if remove_current_maintenance_page or self.user_prompt_callback(
                 f"You are about to enable the '{template}' maintenance page for the {env} "
                 f"environment in {app}.\nWould you like to continue?"
             ):
@@ -68,7 +92,7 @@ class MaintenancePageProvider:
                     allowed_ips,
                     template,
                 )
-                click.secho(
+                self.echo(
                     f"Maintenance page '{template}' added for environment {env} in application {app}",
                     fg="green",
                 )
@@ -76,13 +100,13 @@ class MaintenancePageProvider:
                 raise click.Abort
 
         except LoadBalancerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No load balancer found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
 
         except ListenerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
@@ -96,28 +120,28 @@ class MaintenancePageProvider:
                 application_environment.session, https_listener
             )
             if not current_maintenance_page:
-                click.secho("There is no current maintenance page to remove", fg="red")
+                self.echo("There is no current maintenance page to remove", fg="red")
                 raise click.Abort
 
-            if not click.confirm(
+            if not self.user_prompt_callback(
                 f"There is currently a '{current_maintenance_page}' maintenance page, "
                 f"would you like to remove it?"
             ):
                 raise click.Abort
 
             remove_maintenance_page(application_environment.session, https_listener)
-            click.secho(
+            self.echo(
                 f"Maintenance page removed from environment {env} in application {app}", fg="green"
             )
 
         except LoadBalancerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No load balancer found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort
 
         except ListenerNotFoundException:
-            click.secho(
+            self.echo(
                 f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
             )
             raise click.Abort

dbt_platform_helper/domain/pipelines.py

@@ -0,0 +1,213 @@
+from collections.abc import Callable
+from os import makedirs
+from pathlib import Path
+from shutil import rmtree
+
+from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.utils.application import get_application_name
+from dbt_platform_helper.utils.aws import get_account_details
+from dbt_platform_helper.utils.aws import get_public_repository_arn
+from dbt_platform_helper.utils.files import generate_override_files_from_template
+from dbt_platform_helper.utils.template import setup_templates
+from dbt_platform_helper.utils.versioning import (
+    get_required_terraform_platform_modules_version,
+)
+
+
+class Pipelines:
+    def __init__(
+        self,
+        config_provider: ConfigProvider,
+        echo: Callable[[str], str],
+        abort: Callable[[str], None],
+        get_git_remote: Callable[[], str],
+        get_codestar_arn: Callable[[str], str],
+    ):
+        self.config_provider = config_provider
+        self.echo = echo
+        self.abort = abort
+        self.get_git_remote = get_git_remote
+        self.get_codestar_arn = get_codestar_arn
+
+    def generate(self, terraform_platform_modules_version, deploy_branch):
+        pipeline_config = self.config_provider.load_and_validate_platform_config()
+
+        has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
+        has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
+
+        if not (has_codebase_pipelines or has_environment_pipelines):
+            self.echo("No pipelines defined: nothing to do.", err=True, fg="yellow")
+            return
+
+        platform_config_terraform_modules_default_version = pipeline_config.get(
+            "default_versions", {}
+        ).get("terraform-platform-modules", "")
+
+        templates = setup_templates()
+        app_name = get_application_name()
+
+        git_repo = self.get_git_remote()
+        if not git_repo:
+            self.abort("The current directory is not a git repository")
+
+        codestar_connection_arn = self.get_codestar_arn(app_name)
+        if codestar_connection_arn is None:
+            self.abort(f'There is no CodeStar Connection named "{app_name}" to use')
+
+        base_path = Path(".")
+        copilot_pipelines_dir = base_path / f"copilot/pipelines"
+
+        self._clean_pipeline_config(copilot_pipelines_dir)
+
+        if has_environment_pipelines:
+            environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
+
+            for config in environment_pipelines.values():
+                aws_account = config.get("account")
+                self._generate_terraform_environment_pipeline_manifest(
+                    pipeline_config["application"],
+                    aws_account,
+                    terraform_platform_modules_version,
+                    platform_config_terraform_modules_default_version,
+                    deploy_branch,
+                )
+
+        if has_codebase_pipelines:
+            account_id, _ = get_account_details()
+
+            for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
+                self._generate_codebase_pipeline(
+                    account_id,
+                    app_name,
+                    codestar_connection_arn,
+                    git_repo,
+                    codebase,
+                    base_path,
+                    copilot_pipelines_dir,
+                    templates,
+                )
+
+    def _clean_pipeline_config(self, pipelines_dir):
+        if pipelines_dir.exists():
+            self.echo("Deleting copilot/pipelines directory.")
+            rmtree(pipelines_dir)
+
+    def _generate_codebase_pipeline(
+        self,
+        account_id,
+        app_name,
+        codestar_connection_arn,
+        git_repo,
+        codebase,
+        base_path,
+        pipelines_dir,
+        templates,
+    ):
+        makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
+        environments = []
+        for pipelines in codebase["pipelines"]:
+            environments += pipelines[ENVIRONMENTS_KEY]
+
+        additional_ecr = codebase.get("additional_ecr_repository", None)
+        add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
+        additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
+
+        template_data = {
+            "account_id": account_id,
+            "app_name": app_name,
+            "deploy_repo": git_repo,
+            "codebase": codebase,
+            ENVIRONMENTS_KEY: environments,
+            "codestar_connection_arn": codestar_connection_arn,
+            "codestar_connection_id": codestar_connection_arn.split("/")[-1],
+            "additional_ecr_arn": additional_ecr_arn,
+        }
+
+        self._create_file_from_template(
+            base_path,
+            f"{codebase['name']}/manifest.yml",
+            pipelines_dir,
+            template_data,
+            templates,
+            "codebase/manifest.yml",
+        )
+
+        overrides_path = Path(__file__).parent.parent.joinpath(
+            "templates/pipelines/codebase/overrides"
+        )
+        generate_override_files_from_template(
+            base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
+        )
+
+    def _create_file_from_template(
+        self, base_path, file_name, pipelines_dir, template_data, templates, template_name=None
+    ):
+        contents = templates.get_template(
+            f"pipelines/{file_name if template_name is None else template_name}"
+        ).render(template_data)
+        message = FileProvider.mkfile(
+            base_path, pipelines_dir / file_name, contents, overwrite=True
+        )
+        self.echo(message)
+
+    def _generate_terraform_environment_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
+
+    def generate_terraform_codebase_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("codebase-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))

dbt_platform_helper/domain/terraform_environment.py

@@ -0,0 +1,86 @@
+import click
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.utils.template import setup_templates
+
+
+class PlatformTerraformManifestGenerator:
+    def __init__(self, file_provider):
+        self.file_provider = file_provider
+        self.manifest_template = setup_templates().get_template("environments/main.tf")
+
+    def generate_manifest(
+        self,
+        environment_name: str,
+        application_name: str,
+        environment_config: dict,
+        terraform_platform_modules_version_override: str = None,
+    ):
+        terraform_platform_modules_version = (
+            terraform_platform_modules_version_override
+            or environment_config.get("versions", {}).get(
+                "terraform-platform-modules", DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+            )
+        )
+
+        return self.manifest_template.render(
+            {
+                "application": application_name,
+                "environment": environment_name,
+                "config": environment_config,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+            }
+        )
+
+    def write_manifest(self, environment_name: str, manifest_content: str):
+        return self.file_provider.mkfile(
+            ".",
+            f"terraform/environments/{environment_name}/main.tf",
+            manifest_content,
+            overwrite=True,
+        )
+
+
+class TerraformEnvironmentException(PlatformException):
+    pass
+
+
+class EnvironmentNotFoundException(TerraformEnvironmentException):
+    pass
+
+
+class TerraformEnvironment:
+    def __init__(
+        self,
+        config_provider,
+        manifest_generator: PlatformTerraformManifestGenerator = None,
+        echo_fn=click.echo,
+    ):
+        self.echo = echo_fn
+        self.config_provider = config_provider
+        self.manifest_generator = manifest_generator or PlatformTerraformManifestGenerator(
+            FileProvider()
+        )
+
+    def generate(self, environment_name, terraform_platform_modules_version_override=None):
+        config = self.config_provider.get_enriched_config()
+
+        if environment_name not in config.get("environments").keys():
+            raise EnvironmentNotFoundException(
+                f"Error: cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+            )
+
+        manifest = self.manifest_generator.generate_manifest(
+            environment_name=environment_name,
+            application_name=config["application"],
+            environment_config=config["environments"][environment_name],
+            terraform_platform_modules_version_override=terraform_platform_modules_version_override,
+        )
+
+        self.echo(
+            self.manifest_generator.write_manifest(
+                environment_name=environment_name, manifest_content=manifest
+            )
+        )

dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py

@@ -0,0 +1,100 @@
+from unittest.mock import Mock
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.terraform_environment import (
+    PlatformTerraformManifestGenerator,
+)
+from dbt_platform_helper.providers.files import FileProvider
+
+
+class TestPlatformTerraformManifestGenerator:
+
+    def test_generator_generates_expected_manifest_content_with_version_override(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=123456"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config, 123456
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_tpm_version_set_in_config(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=3"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_default_version(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = f"git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref={DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_write_manifest_makes_the_expected_manifest_file(self):
+        mock_file_provider = Mock(spec=FileProvider)
+        PlatformTerraformManifestGenerator(mock_file_provider).write_manifest(
+            "test-environment", "test-manifest-content"
+        )
+
+        mock_file_provider.mkfile.assert_called_once_with(
+            ".",
+            f"terraform/environments/test-environment/main.tf",
+            "test-manifest-content",
+            overwrite=True,
+        )

dbt_platform_helper/jinja2_tags.py

@@ -17,4 +17,4 @@ class ExtraHeaderTag(StandaloneTag):
     tags = {"extra_header"}
 
     def render(self):
-        return f"WARNING: This is an autogenerated file, not for manual editing."
+        return "WARNING: This is an autogenerated file, not for manual editing."

dbt_platform_helper/providers/cache.py

@@ -1,17 +1,20 @@
 import os
 from datetime import datetime
-from pathlib import Path
 
-import yaml
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
 class CacheProvider:
-    def __init__(self):
+    def __init__(
+        self,
+        file_provider: YamlFileProvider = None,
+    ):
         self._cache_file = ".platform-helper-config-cache.yml"
+        self.file_provider = file_provider or YamlFileProvider
 
     def read_supported_versions_from_cache(self, resource_name):
 
-        platform_helper_config = self.__read_file_as_yaml(self._cache_file)
+        platform_helper_config = self.file_provider.load(self._cache_file)
 
         return platform_helper_config.get(resource_name).get("versions")
 
@@ -20,7 +23,7 @@ class CacheProvider:
         platform_helper_config = {}
 
         if self.__cache_exists():
-            platform_helper_config = self.__read_file_as_yaml(self._cache_file)
+            platform_helper_config = self.file_provider.load(self._cache_file)
 
         cache_dict = {
             resource_name: {
@@ -31,7 +34,11 @@ class CacheProvider:
 
         platform_helper_config.update(cache_dict)
 
-        self.__write_cache(platform_helper_config)
+        self.file_provider.write(
+            self._cache_file,
+            platform_helper_config,
+            "# [!] This file is autogenerated via the platform-helper. Do not edit.\n",
+        )
 
     def cache_refresh_required(self, resource_name) -> bool:
         """
@@ -47,7 +54,7 @@ class CacheProvider:
         if not self.__cache_exists():
             return True
 
-        platform_helper_config = self.__read_file_as_yaml(self._cache_file)
+        platform_helper_config = self.file_provider.load(self._cache_file)
 
         if platform_helper_config.get(resource_name):
             return self.__check_if_cached_datetime_is_greater_than_interval(
@@ -65,19 +72,5 @@ class CacheProvider:
 
         return delta.days > interval_in_days
 
-    # TODO - same applies here as below
-    @staticmethod
-    def __read_file_as_yaml(file_name):
-
-        return yaml.safe_load(Path(file_name).read_text())
-
-    # TODO - temp fix to the unit test coverage issue, plan is to seperate out any yaml interaction methods into a seperate 'yaml' provider
-    # should be done under a different sub-task which will need to loop back to this provider as part of that work to use the yaml provider instead
-    def __write_cache(self, contents):
-
-        with open(self._cache_file, "w") as file:
-            file.write("# [!] This file is autogenerated via the platform-helper. Do not edit.\n")
-            yaml.dump(contents, file)
-
     def __cache_exists(self):
         return os.path.exists(self._cache_file)

dbt_platform_helper/providers/cloudformation.py

@@ -93,7 +93,6 @@ class CloudFormation:
         )
 
         params = []
-        # TODO Currently not covered by tests - see https://uktrade.atlassian.net/browse/DBTP-1582
         if "Parameters" in template_yml:
             for param in template_yml["Parameters"]:
                 params.append({"ParameterKey": param, "UsePreviousValue": True})