dbt-platform-helper 12.5.0__py3-none-any.whl → 12.6.0__py3-none-any.whl

dbt_platform_helper/domain/pipelines.py

@@ -0,0 +1,212 @@
+from collections.abc import Callable
+from os import makedirs
+from pathlib import Path
+from shutil import rmtree
+
+from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.application import get_application_name
+from dbt_platform_helper.utils.aws import get_account_details
+from dbt_platform_helper.utils.aws import get_public_repository_arn
+from dbt_platform_helper.utils.files import generate_override_files_from_template
+from dbt_platform_helper.utils.template import setup_templates
+from dbt_platform_helper.utils.versioning import (
+    get_required_terraform_platform_modules_version,
+)
+
+
+class Pipelines:
+    def __init__(
+        self,
+        config_provider: ConfigProvider,
+        get_git_remote: Callable[[], str],
+        get_codestar_arn: Callable[[str], str],
+        io: ClickIOProvider = ClickIOProvider(),
+    ):
+        self.config_provider = config_provider
+        self.get_git_remote = get_git_remote
+        self.get_codestar_arn = get_codestar_arn
+        self.io = io
+
+    def generate(self, terraform_platform_modules_version, deploy_branch):
+        pipeline_config = self.config_provider.load_and_validate_platform_config()
+
+        has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
+        has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
+
+        if not (has_codebase_pipelines or has_environment_pipelines):
+            self.io.warn("No pipelines defined: nothing to do.")
+            return
+
+        platform_config_terraform_modules_default_version = pipeline_config.get(
+            "default_versions", {}
+        ).get("terraform-platform-modules", "")
+
+        templates = setup_templates()
+        app_name = get_application_name()
+
+        git_repo = self.get_git_remote()
+        if not git_repo:
+            self.io.abort_with_error("The current directory is not a git repository")
+
+        codestar_connection_arn = self.get_codestar_arn(app_name)
+        if codestar_connection_arn is None:
+            self.io.abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
+
+        base_path = Path(".")
+        copilot_pipelines_dir = base_path / f"copilot/pipelines"
+
+        self._clean_pipeline_config(copilot_pipelines_dir)
+
+        if has_environment_pipelines:
+            environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
+
+            for config in environment_pipelines.values():
+                aws_account = config.get("account")
+                self._generate_terraform_environment_pipeline_manifest(
+                    pipeline_config["application"],
+                    aws_account,
+                    terraform_platform_modules_version,
+                    platform_config_terraform_modules_default_version,
+                    deploy_branch,
+                )
+
+        if has_codebase_pipelines:
+            account_id, _ = get_account_details()
+
+            for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
+                self._generate_codebase_pipeline(
+                    account_id,
+                    app_name,
+                    codestar_connection_arn,
+                    git_repo,
+                    codebase,
+                    base_path,
+                    copilot_pipelines_dir,
+                    templates,
+                )
+
+    def _clean_pipeline_config(self, pipelines_dir):
+        if pipelines_dir.exists():
+            self.io.info("Deleting copilot/pipelines directory.")
+            rmtree(pipelines_dir)
+
+    def _generate_codebase_pipeline(
+        self,
+        account_id,
+        app_name,
+        codestar_connection_arn,
+        git_repo,
+        codebase,
+        base_path,
+        pipelines_dir,
+        templates,
+    ):
+        makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
+        environments = []
+        for pipelines in codebase["pipelines"]:
+            environments += pipelines[ENVIRONMENTS_KEY]
+
+        additional_ecr = codebase.get("additional_ecr_repository", None)
+        add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
+        additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
+
+        template_data = {
+            "account_id": account_id,
+            "app_name": app_name,
+            "deploy_repo": git_repo,
+            "codebase": codebase,
+            ENVIRONMENTS_KEY: environments,
+            "codestar_connection_arn": codestar_connection_arn,
+            "codestar_connection_id": codestar_connection_arn.split("/")[-1],
+            "additional_ecr_arn": additional_ecr_arn,
+        }
+
+        self._create_file_from_template(
+            base_path,
+            f"{codebase['name']}/manifest.yml",
+            pipelines_dir,
+            template_data,
+            templates,
+            "codebase/manifest.yml",
+        )
+
+        overrides_path = Path(__file__).parent.parent.joinpath(
+            "templates/pipelines/codebase/overrides"
+        )
+        generate_override_files_from_template(
+            base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
+        )
+
+    def _create_file_from_template(
+        self, base_path, file_name, pipelines_dir, template_data, templates, template_name=None
+    ):
+        contents = templates.get_template(
+            f"pipelines/{file_name if template_name is None else template_name}"
+        ).render(template_data)
+        message = FileProvider.mkfile(
+            base_path, pipelines_dir / file_name, contents, overwrite=True
+        )
+        self.io.info(message)
+
+    def _generate_terraform_environment_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.io.info(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
+
+    def generate_terraform_codebase_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("codebase-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.io.info(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))

dbt_platform_helper/domain/terraform_environment.py

@@ -1,53 +1,86 @@
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
-from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.template import setup_templates
 
 
-def _generate_terraform_environment_manifests(
-    application, env, env_config, cli_terraform_platform_modules_version
-):
-    env_template = setup_templates().get_template("environments/main.tf")
+class PlatformTerraformManifestGenerator:
+    def __init__(self, file_provider):
+        self.file_provider = file_provider
+        self.manifest_template = setup_templates().get_template("environments/main.tf")
 
-    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
-        env_config, cli_terraform_platform_modules_version
-    )
+    def generate_manifest(
+        self,
+        environment_name: str,
+        application_name: str,
+        environment_config: dict,
+        terraform_platform_modules_version_override: str = None,
+    ):
+        terraform_platform_modules_version = (
+            terraform_platform_modules_version_override
+            or environment_config.get("versions", {}).get(
+                "terraform-platform-modules", DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+            )
+        )
+
+        return self.manifest_template.render(
+            {
+                "application": application_name,
+                "environment": environment_name,
+                "config": environment_config,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+            }
+        )
+
+    def write_manifest(self, environment_name: str, manifest_content: str):
+        return self.file_provider.mkfile(
+            ".",
+            f"terraform/environments/{environment_name}/main.tf",
+            manifest_content,
+            overwrite=True,
+        )
 
-    contents = env_template.render(
-        {
-            "application": application,
-            "environment": env,
-            "config": env_config,
-            "terraform_platform_modules_version": terraform_platform_modules_version,
-        }
-    )
 
-    click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
+class TerraformEnvironmentException(PlatformException):
+    pass
 
 
-def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
-    cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
-    env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
-        "terraform-platform-modules"
-    )
-    version_preference_order = [
-        cli_terraform_platform_modules_version,
-        env_conf_terraform_platform_modules_version,
-        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
-    ]
-    return [version for version in version_preference_order if version][0]
+class EnvironmentNotFoundException(TerraformEnvironmentException):
+    pass
 
 
 class TerraformEnvironment:
-    def __init__(self, config_provider):
+    def __init__(
+        self,
+        config_provider,
+        manifest_generator: PlatformTerraformManifestGenerator = None,
+        echo=click.echo,
+    ):
+        self.echo = echo
         self.config_provider = config_provider
+        self.manifest_generator = manifest_generator or PlatformTerraformManifestGenerator(
+            FileProvider()
+        )
+
+    def generate(self, environment_name, terraform_platform_modules_version_override=None):
+        config = self.config_provider.get_enriched_config()
 
-    def generate(self, name, terraform_platform_modules_version):
-        config = self.config_provider.load_and_validate_platform_config()
-        enriched_config = self.config_provider.apply_environment_defaults(config)
+        if environment_name not in config.get("environments").keys():
+            raise EnvironmentNotFoundException(
+                f"cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+            )
+
+        manifest = self.manifest_generator.generate_manifest(
+            environment_name=environment_name,
+            application_name=config["application"],
+            environment_config=config["environments"][environment_name],
+            terraform_platform_modules_version_override=terraform_platform_modules_version_override,
+        )
 
-        env_config = enriched_config["environments"][name]
-        _generate_terraform_environment_manifests(
-            config["application"], name, env_config, terraform_platform_modules_version
+        self.echo(
+            self.manifest_generator.write_manifest(
+                environment_name=environment_name, manifest_content=manifest
+            )
         )

dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py

@@ -0,0 +1,100 @@
+from unittest.mock import Mock
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.terraform_environment import (
+    PlatformTerraformManifestGenerator,
+)
+from dbt_platform_helper.providers.files import FileProvider
+
+
+class TestPlatformTerraformManifestGenerator:
+
+    def test_generator_generates_expected_manifest_content_with_version_override(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=123456"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config, 123456
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_tpm_version_set_in_config(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=3"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_default_version(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = f"git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref={DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_write_manifest_makes_the_expected_manifest_file(self):
+        mock_file_provider = Mock(spec=FileProvider)
+        PlatformTerraformManifestGenerator(mock_file_provider).write_manifest(
+            "test-environment", "test-manifest-content"
+        )
+
+        mock_file_provider.mkfile.assert_called_once_with(
+            ".",
+            f"terraform/environments/test-environment/main.tf",
+            "test-manifest-content",
+            overwrite=True,
+        )

dbt_platform_helper/providers/cache.py

@@ -1,14 +1,13 @@
 import os
 from datetime import datetime
 
-from dbt_platform_helper.providers.yaml_file import FileProvider
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
 class CacheProvider:
     def __init__(
         self,
-        file_provider: FileProvider = None,
+        file_provider: YamlFileProvider = None,
     ):
         self._cache_file = ".platform-helper-config-cache.yml"
         self.file_provider = file_provider or YamlFileProvider

dbt_platform_helper/providers/cloudformation.py

@@ -8,7 +8,8 @@ from dbt_platform_helper.platform_exception import PlatformException
 
 
 class CloudFormation:
-    def __init__(self, cloudformation_client, iam_client, ssm_client):
+    # TODO add handling for optional client parameters to handle case of calling boto API with None
+    def __init__(self, cloudformation_client, iam_client=None, ssm_client=None):
         self.cloudformation_client = cloudformation_client
         self.iam_client = iam_client
         self.ssm_client = ssm_client
@@ -125,6 +126,16 @@ class CloudFormation:
                     stack_name, f"Error while waiting for stack status: {str(err)}"
                 )
 
+    def get_cloudformation_exports_for_environment(self, environment_name):
+        exports = []
+
+        for page in self.cloudformation_client.get_paginator("list_exports").paginate():
+            for export in page["Exports"]:
+                if f"-{environment_name}-" in export["Name"]:
+                    exports.append(export)
+
+        return exports
+
 
 class CloudFormationException(PlatformException):
     def __init__(self, stack_name: str, current_status: str):

dbt_platform_helper/providers/config.py

@@ -1,55 +1,63 @@
 from copy import deepcopy
 from pathlib import Path
 
-import click
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
-from dbt_platform_helper.providers.yaml_file import FileProvider
+from dbt_platform_helper.providers.yaml_file import FileNotFoundException
 from dbt_platform_helper.providers.yaml_file import FileProviderException
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
-from dbt_platform_helper.utils.messages import abort_with_error
 
 
 class ConfigProvider:
     def __init__(
         self,
-        config_validator: ConfigValidator,
-        file_provider: FileProvider = None,
-        echo=click.secho,
+        config_validator: ConfigValidator = None,
+        file_provider: YamlFileProvider = None,
+        io: ClickIOProvider = None,
     ):
         self.config = {}
-        self.validator = config_validator
-        self.echo = echo
+        self.validator = config_validator or ConfigValidator()
+        self.io = io or ClickIOProvider()
         self.file_provider = file_provider or YamlFileProvider
 
+    # TODO refactor so that apply_environment_defaults isn't set, discarded and set again
+    def get_enriched_config(self):
+        return self.apply_environment_defaults(self.load_and_validate_platform_config())
+
     def validate_platform_config(self):
         PlatformConfigSchema.schema().validate(self.config)
 
         # TODO= logically this isn't validation but loading + parsing, to move.
+        # also, we apply defaults but discard that data.  Should we just apply
+        # defaults to config returned by load_and_validate
         enriched_config = ConfigProvider.apply_environment_defaults(self.config)
         self.validator.run_validations(enriched_config)
 
     def load_and_validate_platform_config(self, path=PLATFORM_CONFIG_FILE):
         try:
             self.config = self.file_provider.load(path)
+        except FileNotFoundException as e:
+            self.io.abort_with_error(
+                f"{e} Please check it exists and you are in the root directory of your deployment project."
+            )
         except FileProviderException as e:
-            abort_with_error(f"Error loading configuration from {path}: {e}")
+            self.io.abort_with_error(f"Error loading configuration from {path}: {e}")
 
         try:
             self.validate_platform_config()
         except SchemaError as e:
-            abort_with_error(f"Schema error in {path}. {e}")
+            self.io.abort_with_error(f"Schema error in {path}. {e}")
 
         return self.config
 
-    @staticmethod
     # TODO this general function should be moved out of ConfigProvider
-    def config_file_check(path=PLATFORM_CONFIG_FILE):
+    def config_file_check(self, path=PLATFORM_CONFIG_FILE):
         if not Path(path).exists():
-            abort_with_error(
+            self.io.abort_with_error(
                 f"`{path}` is missing. "
                 "Please check it exists and you are in the root directory of your deployment project."
             )

dbt_platform_helper/providers/copilot.py

@@ -64,6 +64,7 @@ def create_addon_client_task(
         f"{execution_role}"
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--secrets CONNECTION_SECRET={_get_secrets_provider(application, env).get_connection_secret_arn(secret_name)} "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,
@@ -98,6 +99,7 @@ def create_postgres_admin_task(
         f"--task-group-name {task_name} "
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--env-vars CONNECTION_SECRET='{connection_string}' "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,

dbt_platform_helper/providers/files.py

@@ -0,0 +1,26 @@
+from os import makedirs
+from pathlib import Path
+
+
+class FileProvider:
+
+    def load(path: str) -> str:
+        pass
+
+    @staticmethod
+    def mkfile(base_path: str, file_path: str, contents, overwrite=False) -> str:
+        file_path = Path(file_path)
+        file = Path(base_path).joinpath(file_path)
+        file_exists = file.exists()
+
+        if not file_path.parent.exists():
+            makedirs(file_path.parent)
+
+        if file_exists and not overwrite:
+            return f"File {file_path} exists; doing nothing"
+
+        action = "overwritten" if file_exists and overwrite else "created"
+
+        file.write_text(contents)
+
+        return f"File {file_path} {action}"

dbt_platform_helper/providers/io.py

@@ -0,0 +1,31 @@
+import click
+
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ClickIOProvider:
+    def warn(self, message: str):
+        click.secho(message, fg="magenta")
+
+    def error(self, message: str):
+        click.secho(f"Error: {message}", fg="red")
+
+    def info(self, message: str):
+        click.secho(message)
+
+    def input(self, message: str) -> str:
+        return click.prompt(message)
+
+    def confirm(self, message: str) -> bool:
+        try:
+            return click.confirm(message)
+        except click.Abort:
+            raise ClickIOProviderException(message + " [y/N]: Error: invalid input")
+
+    def abort_with_error(self, message: str):
+        click.secho(f"Error: {message}", err=True, fg="red")
+        exit(1)
+
+
+class ClickIOProviderException(PlatformException):
+    pass