dbt-platform-helper 12.5.0__py3-none-any.whl → 12.5.1__py3-none-any.whl

dbt_platform_helper/domain/pipelines.py

@@ -0,0 +1,213 @@
+from collections.abc import Callable
+from os import makedirs
+from pathlib import Path
+from shutil import rmtree
+
+from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.utils.application import get_application_name
+from dbt_platform_helper.utils.aws import get_account_details
+from dbt_platform_helper.utils.aws import get_public_repository_arn
+from dbt_platform_helper.utils.files import generate_override_files_from_template
+from dbt_platform_helper.utils.template import setup_templates
+from dbt_platform_helper.utils.versioning import (
+    get_required_terraform_platform_modules_version,
+)
+
+
+class Pipelines:
+    def __init__(
+        self,
+        config_provider: ConfigProvider,
+        echo: Callable[[str], str],
+        abort: Callable[[str], None],
+        get_git_remote: Callable[[], str],
+        get_codestar_arn: Callable[[str], str],
+    ):
+        self.config_provider = config_provider
+        self.echo = echo
+        self.abort = abort
+        self.get_git_remote = get_git_remote
+        self.get_codestar_arn = get_codestar_arn
+
+    def generate(self, terraform_platform_modules_version, deploy_branch):
+        pipeline_config = self.config_provider.load_and_validate_platform_config()
+
+        has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
+        has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
+
+        if not (has_codebase_pipelines or has_environment_pipelines):
+            self.echo("No pipelines defined: nothing to do.", err=True, fg="yellow")
+            return
+
+        platform_config_terraform_modules_default_version = pipeline_config.get(
+            "default_versions", {}
+        ).get("terraform-platform-modules", "")
+
+        templates = setup_templates()
+        app_name = get_application_name()
+
+        git_repo = self.get_git_remote()
+        if not git_repo:
+            self.abort("The current directory is not a git repository")
+
+        codestar_connection_arn = self.get_codestar_arn(app_name)
+        if codestar_connection_arn is None:
+            self.abort(f'There is no CodeStar Connection named "{app_name}" to use')
+
+        base_path = Path(".")
+        copilot_pipelines_dir = base_path / f"copilot/pipelines"
+
+        self._clean_pipeline_config(copilot_pipelines_dir)
+
+        if has_environment_pipelines:
+            environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
+
+            for config in environment_pipelines.values():
+                aws_account = config.get("account")
+                self._generate_terraform_environment_pipeline_manifest(
+                    pipeline_config["application"],
+                    aws_account,
+                    terraform_platform_modules_version,
+                    platform_config_terraform_modules_default_version,
+                    deploy_branch,
+                )
+
+        if has_codebase_pipelines:
+            account_id, _ = get_account_details()
+
+            for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
+                self._generate_codebase_pipeline(
+                    account_id,
+                    app_name,
+                    codestar_connection_arn,
+                    git_repo,
+                    codebase,
+                    base_path,
+                    copilot_pipelines_dir,
+                    templates,
+                )
+
+    def _clean_pipeline_config(self, pipelines_dir):
+        if pipelines_dir.exists():
+            self.echo("Deleting copilot/pipelines directory.")
+            rmtree(pipelines_dir)
+
+    def _generate_codebase_pipeline(
+        self,
+        account_id,
+        app_name,
+        codestar_connection_arn,
+        git_repo,
+        codebase,
+        base_path,
+        pipelines_dir,
+        templates,
+    ):
+        makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
+        environments = []
+        for pipelines in codebase["pipelines"]:
+            environments += pipelines[ENVIRONMENTS_KEY]
+
+        additional_ecr = codebase.get("additional_ecr_repository", None)
+        add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
+        additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
+
+        template_data = {
+            "account_id": account_id,
+            "app_name": app_name,
+            "deploy_repo": git_repo,
+            "codebase": codebase,
+            ENVIRONMENTS_KEY: environments,
+            "codestar_connection_arn": codestar_connection_arn,
+            "codestar_connection_id": codestar_connection_arn.split("/")[-1],
+            "additional_ecr_arn": additional_ecr_arn,
+        }
+
+        self._create_file_from_template(
+            base_path,
+            f"{codebase['name']}/manifest.yml",
+            pipelines_dir,
+            template_data,
+            templates,
+            "codebase/manifest.yml",
+        )
+
+        overrides_path = Path(__file__).parent.parent.joinpath(
+            "templates/pipelines/codebase/overrides"
+        )
+        generate_override_files_from_template(
+            base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
+        )
+
+    def _create_file_from_template(
+        self, base_path, file_name, pipelines_dir, template_data, templates, template_name=None
+    ):
+        contents = templates.get_template(
+            f"pipelines/{file_name if template_name is None else template_name}"
+        ).render(template_data)
+        message = FileProvider.mkfile(
+            base_path, pipelines_dir / file_name, contents, overwrite=True
+        )
+        self.echo(message)
+
+    def _generate_terraform_environment_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
+
+    def generate_terraform_codebase_pipeline_manifest(
+        self,
+        application,
+        aws_account,
+        cli_terraform_platform_modules_version,
+        platform_config_terraform_modules_default_version,
+        deploy_branch,
+    ):
+        env_pipeline_template = setup_templates().get_template("codebase-pipelines/main.tf")
+
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            cli_terraform_platform_modules_version,
+            platform_config_terraform_modules_default_version,
+        )
+
+        contents = env_pipeline_template.render(
+            {
+                "application": application,
+                "aws_account": aws_account,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "deploy_branch": deploy_branch,
+            }
+        )
+
+        dir_path = f"terraform/environment-pipelines/{aws_account}"
+        makedirs(dir_path, exist_ok=True)
+
+        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))

dbt_platform_helper/domain/terraform_environment.py

@@ -1,53 +1,86 @@
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
-from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.utils.template import setup_templates
 
 
-def _generate_terraform_environment_manifests(
-    application, env, env_config, cli_terraform_platform_modules_version
-):
-    env_template = setup_templates().get_template("environments/main.tf")
+class PlatformTerraformManifestGenerator:
+    def __init__(self, file_provider):
+        self.file_provider = file_provider
+        self.manifest_template = setup_templates().get_template("environments/main.tf")
 
-    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
-        env_config, cli_terraform_platform_modules_version
-    )
+    def generate_manifest(
+        self,
+        environment_name: str,
+        application_name: str,
+        environment_config: dict,
+        terraform_platform_modules_version_override: str = None,
+    ):
+        terraform_platform_modules_version = (
+            terraform_platform_modules_version_override
+            or environment_config.get("versions", {}).get(
+                "terraform-platform-modules", DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+            )
+        )
+
+        return self.manifest_template.render(
+            {
+                "application": application_name,
+                "environment": environment_name,
+                "config": environment_config,
+                "terraform_platform_modules_version": terraform_platform_modules_version,
+            }
+        )
+
+    def write_manifest(self, environment_name: str, manifest_content: str):
+        return self.file_provider.mkfile(
+            ".",
+            f"terraform/environments/{environment_name}/main.tf",
+            manifest_content,
+            overwrite=True,
+        )
 
-    contents = env_template.render(
-        {
-            "application": application,
-            "environment": env,
-            "config": env_config,
-            "terraform_platform_modules_version": terraform_platform_modules_version,
-        }
-    )
 
-    click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
+class TerraformEnvironmentException(PlatformException):
+    pass
 
 
-def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
-    cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
-    env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
-        "terraform-platform-modules"
-    )
-    version_preference_order = [
-        cli_terraform_platform_modules_version,
-        env_conf_terraform_platform_modules_version,
-        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
-    ]
-    return [version for version in version_preference_order if version][0]
+class EnvironmentNotFoundException(TerraformEnvironmentException):
+    pass
 
 
 class TerraformEnvironment:
-    def __init__(self, config_provider):
+    def __init__(
+        self,
+        config_provider,
+        manifest_generator: PlatformTerraformManifestGenerator = None,
+        echo_fn=click.echo,
+    ):
+        self.echo = echo_fn
         self.config_provider = config_provider
+        self.manifest_generator = manifest_generator or PlatformTerraformManifestGenerator(
+            FileProvider()
+        )
+
+    def generate(self, environment_name, terraform_platform_modules_version_override=None):
+        config = self.config_provider.get_enriched_config()
 
-    def generate(self, name, terraform_platform_modules_version):
-        config = self.config_provider.load_and_validate_platform_config()
-        enriched_config = self.config_provider.apply_environment_defaults(config)
+        if environment_name not in config.get("environments").keys():
+            raise EnvironmentNotFoundException(
+                f"Error: cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+            )
+
+        manifest = self.manifest_generator.generate_manifest(
+            environment_name=environment_name,
+            application_name=config["application"],
+            environment_config=config["environments"][environment_name],
+            terraform_platform_modules_version_override=terraform_platform_modules_version_override,
+        )
 
-        env_config = enriched_config["environments"][name]
-        _generate_terraform_environment_manifests(
-            config["application"], name, env_config, terraform_platform_modules_version
+        self.echo(
+            self.manifest_generator.write_manifest(
+                environment_name=environment_name, manifest_content=manifest
+            )
         )

dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py

@@ -0,0 +1,100 @@
+from unittest.mock import Mock
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.terraform_environment import (
+    PlatformTerraformManifestGenerator,
+)
+from dbt_platform_helper.providers.files import FileProvider
+
+
+class TestPlatformTerraformManifestGenerator:
+
+    def test_generator_generates_expected_manifest_content_with_version_override(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=123456"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config, 123456
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_tpm_version_set_in_config(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+            "versions": {"terraform-platform-modules": 3},
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = "git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref=3"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_generates_expected_manifest_content_with_default_version(self):
+        test_environment_config = {
+            "vpc": "vpc3",
+            "accounts": {
+                "deploy": {"name": "non-prod-acc", "id": "1122334455"},
+                "dns": {"name": "non-prod-dns-acc", "id": "6677889900"},
+            },
+        }
+        expected_header = "# WARNING: This is an autogenerated file, not for manual editing."
+        expected_modules = f"git::https://github.com/uktrade/terraform-platform-modules.git//extensions?depth=1&ref={DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}"
+        expected_moved_block = (
+            "moved {\n  from = module.extensions-tf\n  to   = module.extensions\n}\n"
+        )
+
+        result = PlatformTerraformManifestGenerator(Mock()).generate_manifest(
+            "test", "test-app", test_environment_config
+        )
+
+        assert expected_header in result
+        assert expected_modules in result
+        assert expected_moved_block in result
+        assert 'environment = "test"' in result
+        assert 'application    = "test-app"' in result
+        assert 'vpc_name    = "vpc3"'
+
+    def test_generator_write_manifest_makes_the_expected_manifest_file(self):
+        mock_file_provider = Mock(spec=FileProvider)
+        PlatformTerraformManifestGenerator(mock_file_provider).write_manifest(
+            "test-environment", "test-manifest-content"
+        )
+
+        mock_file_provider.mkfile.assert_called_once_with(
+            ".",
+            f"terraform/environments/test-environment/main.tf",
+            "test-manifest-content",
+            overwrite=True,
+        )

dbt_platform_helper/providers/cache.py

@@ -1,14 +1,13 @@
 import os
 from datetime import datetime
 
-from dbt_platform_helper.providers.yaml_file import FileProvider
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 
 
 class CacheProvider:
     def __init__(
         self,
-        file_provider: FileProvider = None,
+        file_provider: YamlFileProvider = None,
     ):
         self._cache_file = ".platform-helper-config-cache.yml"
         self.file_provider = file_provider or YamlFileProvider

dbt_platform_helper/providers/config.py

@@ -7,7 +7,7 @@ from schema import SchemaError
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
-from dbt_platform_helper.providers.yaml_file import FileProvider
+from dbt_platform_helper.providers.yaml_file import FileNotFoundException
 from dbt_platform_helper.providers.yaml_file import FileProviderException
 from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 from dbt_platform_helper.utils.messages import abort_with_error
@@ -17,7 +17,7 @@ class ConfigProvider:
     def __init__(
         self,
         config_validator: ConfigValidator,
-        file_provider: FileProvider = None,
+        file_provider: YamlFileProvider = None,
         echo=click.secho,
     ):
         self.config = {}
@@ -25,16 +25,26 @@ class ConfigProvider:
         self.echo = echo
         self.file_provider = file_provider or YamlFileProvider
 
+    # TODO refactor so that apply_environment_defaults isn't set, discarded and set again
+    def get_enriched_config(self):
+        return self.apply_environment_defaults(self.load_and_validate_platform_config())
+
     def validate_platform_config(self):
         PlatformConfigSchema.schema().validate(self.config)
 
         # TODO= logically this isn't validation but loading + parsing, to move.
+        # also, we apply defaults but discard that data.  Should we just apply
+        # defaults to config returned by load_and_validate
         enriched_config = ConfigProvider.apply_environment_defaults(self.config)
         self.validator.run_validations(enriched_config)
 
     def load_and_validate_platform_config(self, path=PLATFORM_CONFIG_FILE):
         try:
             self.config = self.file_provider.load(path)
+        except FileNotFoundException as e:
+            abort_with_error(
+                f"{e} Please check it exists and you are in the root directory of your deployment project."
+            )
         except FileProviderException as e:
             abort_with_error(f"Error loading configuration from {path}: {e}")
 

dbt_platform_helper/providers/copilot.py

@@ -64,6 +64,7 @@ def create_addon_client_task(
         f"{execution_role}"
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--secrets CONNECTION_SECRET={_get_secrets_provider(application, env).get_connection_secret_arn(secret_name)} "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,
@@ -98,6 +99,7 @@ def create_postgres_admin_task(
         f"--task-group-name {task_name} "
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
         f"--env-vars CONNECTION_SECRET='{connection_string}' "
+        "--cpu 2048 --memory 4096 "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,

dbt_platform_helper/providers/files.py

@@ -0,0 +1,26 @@
+from os import makedirs
+from pathlib import Path
+
+
+class FileProvider:
+
+    def load(path: str) -> str:
+        pass
+
+    @staticmethod
+    def mkfile(base_path: str, file_path: str, contents, overwrite=False) -> str:
+        file_path = Path(file_path)
+        file = Path(base_path).joinpath(file_path)
+        file_exists = file.exists()
+
+        if not file_path.parent.exists():
+            makedirs(file_path.parent)
+
+        if file_exists and not overwrite:
+            return f"File {file_path} exists; doing nothing"
+
+        action = "overwritten" if file_exists and overwrite else "created"
+
+        file.write_text(contents)
+
+        return f"File {file_path} {action}"

dbt_platform_helper/providers/vpc.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+
+from dbt_platform_helper.providers.aws import AWSException
+
+
+@dataclass
+class Vpc:
+    subnets: list[str]
+    security_groups: list[str]
+
+
+class VpcProvider:
+    def __init__(self, session):
+        self.ec2_client = session.client("ec2")
+        self.ec2_resource = session.resource("ec2")
+
+    def get_vpc_info_by_name(self, app: str, env: str, vpc_name: str) -> Vpc:
+        vpc_response = self.ec2_client.describe_vpcs(
+            Filters=[{"Name": "tag:Name", "Values": [vpc_name]}]
+        )
+
+        matching_vpcs = vpc_response.get("Vpcs", [])
+
+        if not matching_vpcs:
+            raise AWSException(f"VPC not found for name '{vpc_name}'")
+
+        vpc_id = vpc_response["Vpcs"][0].get("VpcId")
+
+        if not vpc_id:
+            raise AWSException(f"VPC id not present in vpc '{vpc_name}'")
+
+        vpc = self.ec2_resource.Vpc(vpc_id)
+
+        route_tables = self.ec2_client.describe_route_tables(
+            Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+        )["RouteTables"]
+
+        subnets = []
+        for route_table in route_tables:
+            private_routes = [route for route in route_table["Routes"] if "NatGatewayId" in route]
+            if not private_routes:
+                continue
+            for association in route_table["Associations"]:
+                if "SubnetId" in association:
+                    subnet_id = association["SubnetId"]
+                    subnets.append(subnet_id)
+
+        if not subnets:
+            raise AWSException(f"No private subnets found in vpc '{vpc_name}'")
+
+        tag_value = {"Key": "Name", "Value": f"copilot-{app}-{env}-env"}
+        sec_groups = [sg.id for sg in vpc.security_groups.all() if sg.tags and tag_value in sg.tags]
+
+        if not sec_groups:
+            raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
+
+        return Vpc(subnets, sec_groups)

dbt_platform_helper/providers/yaml_file.py

@@ -1,5 +1,3 @@
-from abc import ABC
-from abc import abstractmethod
 from pathlib import Path
 
 import yaml
@@ -16,7 +14,7 @@ class YamlFileProviderException(FileProviderException):
     pass
 
 
-class FileNotFoundException(YamlFileProviderException):
+class FileNotFoundException(FileProviderException):
     pass
 
 
@@ -28,13 +26,7 @@ class DuplicateKeysException(YamlFileProviderException):
     pass
 
 
-class FileProvider(ABC):
-    @abstractmethod
-    def load(path: str) -> dict:
-        raise NotImplementedError("Implement this in the subclass")
-
-
-class YamlFileProvider(FileProvider):
+class YamlFileProvider:
     def load(path: str) -> dict:
         """
         Raises:
@@ -43,10 +35,7 @@ class YamlFileProvider(FileProvider):
             DuplicateKeysException: yaml contains duplicate keys
         """
         if not Path(path).exists():
-            # TODO this error message is domain specific and should not mention deployment directory project here
-            raise FileNotFoundException(
-                f"`{path}` is missing. Please check it exists and you are in the root directory of your deployment project."
-            )
+            raise FileNotFoundException(f"`{path}` is missing.")
         try:
             yaml_content = yaml.safe_load(Path(path).read_text())
         except ParserError: