dbt-platform-helper 12.4.0__py3-none-any.whl → 12.5.0__py3-none-any.whl

dbt_platform_helper/domain/config_validator.py

@@ -0,0 +1,242 @@
+from typing import Callable
+
+import boto3
+import click
+
+from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.providers.opensearch import OpensearchProvider
+from dbt_platform_helper.providers.redis import RedisProvider
+from dbt_platform_helper.utils.messages import abort_with_error
+
+
+class ConfigValidator:
+
+    def __init__(self, validations: Callable[[dict], None] = None):
+        self.validations = validations or [
+            self.validate_supported_redis_versions,
+            self.validate_supported_opensearch_versions,
+            self.validate_environment_pipelines,
+            self.validate_codebase_pipelines,
+            self.validate_environment_pipelines_triggers,
+            self.validate_database_copy_section,
+        ]
+
+    def run_validations(self, config: dict):
+        for validation in self.validations:
+            validation(config)
+
+    def _validate_extension_supported_versions(
+        self, config, extension_type, version_key, get_supported_versions
+    ):
+        extensions = config.get("extensions", {})
+        if not extensions:
+            return
+
+        extensions_for_type = [
+            extension
+            for extension in config.get("extensions", {}).values()
+            if extension.get("type") == extension_type
+        ]
+
+        supported_extension_versions = get_supported_versions()
+        extensions_with_invalid_version = []
+
+        for extension in extensions_for_type:
+
+            environments = extension.get("environments", {})
+
+            if not isinstance(environments, dict):
+                click.secho(
+                    f"Error: {extension_type} extension definition is invalid type, expected dictionary",
+                    fg="red",
+                )
+                continue
+            for environment, env_config in environments.items():
+
+                # An extension version doesn't need to be specified for all environments, provided one is specified under "*".
+                # So check if the version is set before checking if it's supported
+                extension_version = env_config.get(version_key)
+                if extension_version and extension_version not in supported_extension_versions:
+                    extensions_with_invalid_version.append(
+                        {"environment": environment, "version": extension_version}
+                    )
+
+        for version_failure in extensions_with_invalid_version:
+            click.secho(
+                f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
+                fg="red",
+            )
+
+    def validate_supported_redis_versions(self, config):
+        return self._validate_extension_supported_versions(
+            config=config,
+            extension_type="redis",
+            version_key="engine",
+            get_supported_versions=RedisProvider(
+                boto3.client("elasticache")
+            ).get_supported_redis_versions,
+        )
+
+    def validate_supported_opensearch_versions(self, config):
+        return self._validate_extension_supported_versions(
+            config=config,
+            extension_type="opensearch",
+            version_key="engine",
+            get_supported_versions=OpensearchProvider(
+                boto3.client("opensearch")
+            ).get_supported_opensearch_versions,
+        )
+
+    def validate_environment_pipelines(self, config):
+        bad_pipelines = {}
+        for pipeline_name, pipeline in config.get("environment_pipelines", {}).items():
+            bad_envs = []
+            pipeline_account = pipeline.get("account", None)
+            if pipeline_account:
+                for env in pipeline.get("environments", {}).keys():
+                    env_account = (
+                        config.get("environments", {})
+                        .get(env, {})
+                        .get("accounts", {})
+                        .get("deploy", {})
+                        .get("name")
+                    )
+                    if not env_account == pipeline_account:
+                        bad_envs.append(env)
+            if bad_envs:
+                bad_pipelines[pipeline_name] = {"account": pipeline_account, "bad_envs": bad_envs}
+        if bad_pipelines:
+            message = "The following pipelines are misconfigured:"
+            for pipeline, detail in bad_pipelines.items():
+                envs = detail["bad_envs"]
+                acc = detail["account"]
+                message += f"  '{pipeline}' - these environments are not in the '{acc}' account: {', '.join(envs)}\n"
+            abort_with_error(message)
+
+    def validate_codebase_pipelines(self, config):
+        if CODEBASE_PIPELINES_KEY in config:
+            for codebase in config[CODEBASE_PIPELINES_KEY]:
+                codebase_environments = []
+
+                for pipeline in codebase["pipelines"]:
+                    codebase_environments += [e["name"] for e in pipeline[ENVIRONMENTS_KEY]]
+
+                unique_codebase_environments = sorted(list(set(codebase_environments)))
+
+                if sorted(codebase_environments) != sorted(unique_codebase_environments):
+                    abort_with_error(
+                        f"The {PLATFORM_CONFIG_FILE} file is invalid, each environment can only be "
+                        "listed in a single pipeline per codebase"
+                    )
+
+    def validate_environment_pipelines_triggers(self, config):
+        errors = []
+        pipelines_with_triggers = {
+            pipeline_name: pipeline
+            for pipeline_name, pipeline in config.get("environment_pipelines", {}).items()
+            if "pipeline_to_trigger" in pipeline
+        }
+
+        for pipeline_name, pipeline in pipelines_with_triggers.items():
+            pipeline_to_trigger = pipeline["pipeline_to_trigger"]
+            if pipeline_to_trigger not in config.get("environment_pipelines", {}):
+                message = f"  '{pipeline_name}' - '{pipeline_to_trigger}' is not a valid target pipeline to trigger"
+
+                errors.append(message)
+                continue
+
+            if pipeline_to_trigger == pipeline_name:
+                message = f"  '{pipeline_name}' - pipelines cannot trigger themselves"
+                errors.append(message)
+
+        if errors:
+            error_message = "The following pipelines are misconfigured: \n"
+            abort_with_error(error_message + "\n  ".join(errors))
+
+    def validate_database_copy_section(self, config):
+        extensions = config.get("extensions", {})
+        if not extensions:
+            return
+
+        postgres_extensions = {
+            key: ext for key, ext in extensions.items() if ext.get("type", None) == "postgres"
+        }
+
+        if not postgres_extensions:
+            return
+
+        errors = []
+
+        for extension_name, extension in postgres_extensions.items():
+            database_copy_sections = extension.get("database_copy", [])
+
+            if not database_copy_sections:
+                return
+
+            all_environments = [
+                env for env in config.get("environments", {}).keys() if not env == "*"
+            ]
+            all_envs_string = ", ".join(all_environments)
+
+            for section in database_copy_sections:
+                from_env = section["from"]
+                to_env = section["to"]
+
+                from_account = (
+                    config.get("environments", {})
+                    .get(from_env, {})
+                    .get("accounts", {})
+                    .get("deploy", {})
+                    .get("id")
+                )
+                to_account = (
+                    config.get("environments", {})
+                    .get(to_env, {})
+                    .get("accounts", {})
+                    .get("deploy", {})
+                    .get("id")
+                )
+
+                if from_env == to_env:
+                    errors.append(
+                        f"database_copy 'to' and 'from' cannot be the same environment in extension '{extension_name}'."
+                    )
+
+                if "prod" in to_env:
+                    errors.append(
+                        f"Copying to a prod environment is not supported: database_copy 'to' cannot be '{to_env}' in extension '{extension_name}'."
+                    )
+
+                if from_env not in all_environments:
+                    errors.append(
+                        f"database_copy 'from' parameter must be a valid environment ({all_envs_string}) but was '{from_env}' in extension '{extension_name}'."
+                    )
+
+                if to_env not in all_environments:
+                    errors.append(
+                        f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
+                    )
+
+                if from_account != to_account:
+                    if "from_account" not in section:
+                        errors.append(
+                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'from_account' parameter must be present."
+                        )
+                    elif section["from_account"] != from_account:
+                        errors.append(
+                            f"Incorrect value for 'from_account' for environment '{from_env}'"
+                        )
+
+                    if "to_account" not in section:
+                        errors.append(
+                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'to_account' parameter must be present."
+                        )
+                    elif section["to_account"] != to_account:
+                        errors.append(
+                            f"Incorrect value for 'to_account' for environment '{to_env}'"
+                        )
+
+        if errors:
+            abort_with_error("\n".join(errors))

dbt_platform_helper/domain/copilot_environment.py

@@ -0,0 +1,204 @@
+from collections import defaultdict
+from pathlib import Path
+
+import boto3
+import click
+
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.template import S3_CROSS_ACCOUNT_POLICY
+from dbt_platform_helper.utils.template import camel_case
+from dbt_platform_helper.utils.template import setup_templates
+
+
+# TODO - move helper functions into suitable provider classes
+def get_subnet_ids(session, vpc_id, environment_name):
+    subnets = session.client("ec2").describe_subnets(
+        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+    )["Subnets"]
+
+    if not subnets:
+        click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
+        raise click.Abort
+
+    public_tag = {"Key": "subnet_type", "Value": "public"}
+    public_subnets = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
+    private_tag = {"Key": "subnet_type", "Value": "private"}
+    private_subnets = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
+
+    # This call and the method declaration can be removed when we stop using AWS Copilot to deploy the services
+    public_subnets, private_subnets = _match_subnet_id_order_to_cloudformation_exports(
+        session,
+        environment_name,
+        public_subnets,
+        private_subnets,
+    )
+
+    return public_subnets, private_subnets
+
+
+def _match_subnet_id_order_to_cloudformation_exports(
+    session, environment_name, public_subnets, private_subnets
+):
+    public_subnet_exports = []
+    private_subnet_exports = []
+    for page in session.client("cloudformation").get_paginator("list_exports").paginate():
+        for export in page["Exports"]:
+            if f"-{environment_name}-" in export["Name"]:
+                if export["Name"].endswith("-PublicSubnets"):
+                    public_subnet_exports = export["Value"].split(",")
+                if export["Name"].endswith("-PrivateSubnets"):
+                    private_subnet_exports = export["Value"].split(",")
+
+    # If the elements match, regardless of order, use the list from the CloudFormation exports
+    if set(public_subnets) == set(public_subnet_exports):
+        public_subnets = public_subnet_exports
+    if set(private_subnets) == set(private_subnet_exports):
+        private_subnets = private_subnet_exports
+
+    return public_subnets, private_subnets
+
+
+def get_cert_arn(session, application, env_name):
+    try:
+        arn = find_https_certificate(session, application, env_name)
+    except:
+        click.secho(
+            f"No certificate found with domain name matching environment {env_name}.", fg="red"
+        )
+        raise click.Abort
+
+    return arn
+
+
+def get_vpc_id(session, env_name, vpc_name=None):
+    if not vpc_name:
+        vpc_name = f"{session.profile_name}-{env_name}"
+
+    filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
+    vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+
+    if not vpcs:
+        filters[0]["Values"] = [session.profile_name]
+        vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+
+    if not vpcs:
+        click.secho(
+            f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
+        )
+        raise click.Abort
+
+    return vpcs[0]["VpcId"]
+
+
+def _generate_copilot_environment_manifests(
+    environment_name, application_name, env_config, session
+):
+    env_template = setup_templates().get_template("env/manifest.yml")
+    vpc_name = env_config.get("vpc", None)
+    vpc_id = get_vpc_id(session, environment_name, vpc_name)
+    pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id, environment_name)
+    cert_arn = get_cert_arn(session, application_name, environment_name)
+    contents = env_template.render(
+        {
+            "name": environment_name,
+            "vpc_id": vpc_id,
+            "pub_subnet_ids": pub_subnet_ids,
+            "priv_subnet_ids": priv_subnet_ids,
+            "certificate_arn": cert_arn,
+        }
+    )
+    click.echo(
+        mkfile(
+            ".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
+        )
+    )
+
+
+def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
+    listener_arn = find_https_listener(session, app, env)
+    cert_client = session.client("elbv2")
+    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
+        "Certificates"
+    ]
+
+    try:
+        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+    except StopIteration:
+        raise CertificateNotFoundException()
+
+    return certificate_arn
+
+
+class CertificateNotFoundException(PlatformException):
+    pass
+
+
+class CopilotEnvironment:
+    def __init__(self, config_provider):
+        self.config_provider = config_provider
+
+    def generate(self, environment_name):
+        config = self.config_provider.load_and_validate_platform_config()
+        enriched_config = self.config_provider.apply_environment_defaults(config)
+
+        env_config = enriched_config["environments"][environment_name]
+        profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
+        click.secho(f"Using {profile_for_environment} for this AWS session")
+        session = get_aws_session_or_abort(profile_for_environment)
+
+        _generate_copilot_environment_manifests(
+            environment_name, enriched_config["application"], env_config, session
+        )
+
+
+class CopilotTemplating:
+    def __init__(self, mkfile_fn=mkfile):
+        self.mkfile_fn = mkfile_fn
+
+    def generate_cross_account_s3_policies(self, environments: dict, extensions):
+        resource_blocks = defaultdict(list)
+
+        for ext_name, ext_data in extensions.items():
+            for env_name, env_data in ext_data.get("environments", {}).items():
+                if "cross_environment_service_access" in env_data:
+                    bucket = env_data.get("bucket_name")
+                    x_env_data = env_data["cross_environment_service_access"]
+                    for access_name, access_data in x_env_data.items():
+                        service = access_data.get("service")
+                        read = access_data.get("read", False)
+                        write = access_data.get("write", False)
+                        if read or write:
+                            resource_blocks[service].append(
+                                {
+                                    "bucket_name": bucket,
+                                    "app_prefix": camel_case(f"{service}-{bucket}-{access_name}"),
+                                    "bucket_env": env_name,
+                                    "access_env": access_data.get("environment"),
+                                    "bucket_account": environments.get(env_name, {})
+                                    .get("accounts", {})
+                                    .get("deploy", {})
+                                    .get("id"),
+                                    "read": read,
+                                    "write": write,
+                                }
+                            )
+
+        if not resource_blocks:
+            click.echo("\n>>> No cross-environment S3 policies to create.\n")
+            return
+
+        templates = setup_templates()
+
+        for service in sorted(resource_blocks.keys()):
+            resources = resource_blocks[service]
+            click.echo(f"\n>>> Creating S3 cross account policies for {service}.\n")
+            template = templates.get_template(S3_CROSS_ACCOUNT_POLICY)
+            file_content = template.render({"resources": resources})
+            output_dir = Path(".").absolute()
+            file_path = f"copilot/{service}/addons/s3-cross-account-policy.yml"
+
+            self.mkfile_fn(output_dir, file_path, file_content, True)
+            click.echo(f"File {file_path} created")

dbt_platform_helper/domain/database_copy.py

@@ -7,8 +7,10 @@ import click
 from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
 from dbt_platform_helper.providers.aws import AWSException
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationNotFoundException
 from dbt_platform_helper.utils.application import load_application
@@ -16,9 +18,7 @@ from dbt_platform_helper.utils.aws import Vpc
 from dbt_platform_helper.utils.aws import get_connection_string
 from dbt_platform_helper.utils.aws import get_vpc_info_by_name
 from dbt_platform_helper.utils.aws import wait_for_log_group_to_exist
-from dbt_platform_helper.utils.files import apply_environment_defaults
 from dbt_platform_helper.utils.messages import abort_with_error
-from dbt_platform_helper.utils.validation import load_and_validate_platform_config
 
 
 class DatabaseCopy:
@@ -38,6 +38,7 @@ class DatabaseCopy:
         input: Callable[[str], str] = click.prompt,
         echo: Callable[[str], str] = click.secho,
         abort: Callable[[str], None] = abort_with_error,
+        config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
     ):
         self.app = app
         self.database = database
@@ -48,12 +49,13 @@ class DatabaseCopy:
         self.input = input
         self.echo = echo
         self.abort = abort
+        self.config_provider = config_provider
 
         if not self.app:
             if not Path(PLATFORM_CONFIG_FILE).exists():
                 self.abort("You must either be in a deploy repo, or provide the --app option.")
 
-            config = load_and_validate_platform_config()
+            config = self.config_provider.load_and_validate_platform_config()
             self.app = config["application"]
 
         try:
@@ -110,8 +112,8 @@ class DatabaseCopy:
         if not vpc_name:
             if not Path(PLATFORM_CONFIG_FILE).exists():
                 self.abort("You must either be in a deploy repo, or provide the vpc name option.")
-            config = load_and_validate_platform_config()
-            env_config = apply_environment_defaults(config)["environments"]
+            config = self.config_provider.load_and_validate_platform_config()
+            env_config = self.config_provider.apply_environment_defaults(config)["environments"]
             vpc_name = env_config.get(env, {}).get("vpc")
         return vpc_name
 

dbt_platform_helper/domain/maintenance_page.py

@@ -261,7 +261,7 @@ def add_maintenance_page(
         )
 
         click.secho(
-            f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/activities/holding-and-maintenance-pages/",
+            f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
             fg="green",
         )
 

dbt_platform_helper/domain/terraform_environment.py

@@ -0,0 +1,53 @@
+import click
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.template import setup_templates
+
+
+def _generate_terraform_environment_manifests(
+    application, env, env_config, cli_terraform_platform_modules_version
+):
+    env_template = setup_templates().get_template("environments/main.tf")
+
+    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
+        env_config, cli_terraform_platform_modules_version
+    )
+
+    contents = env_template.render(
+        {
+            "application": application,
+            "environment": env,
+            "config": env_config,
+            "terraform_platform_modules_version": terraform_platform_modules_version,
+        }
+    )
+
+    click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
+
+
+def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
+    cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
+    env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
+        "terraform-platform-modules"
+    )
+    version_preference_order = [
+        cli_terraform_platform_modules_version,
+        env_conf_terraform_platform_modules_version,
+        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
+    ]
+    return [version for version in version_preference_order if version][0]
+
+
+class TerraformEnvironment:
+    def __init__(self, config_provider):
+        self.config_provider = config_provider
+
+    def generate(self, name, terraform_platform_modules_version):
+        config = self.config_provider.load_and_validate_platform_config()
+        enriched_config = self.config_provider.apply_environment_defaults(config)
+
+        env_config = enriched_config["environments"][name]
+        _generate_terraform_environment_manifests(
+            config["application"], name, env_config, terraform_platform_modules_version
+        )

dbt_platform_helper/jinja2_tags.py

@@ -17,4 +17,4 @@ class ExtraHeaderTag(StandaloneTag):
     tags = {"extra_header"}
 
     def render(self):
-        return f"WARNING: This is an autogenerated file, not for manual editing."
+        return "WARNING: This is an autogenerated file, not for manual editing."

dbt_platform_helper/providers/cache.py

@@ -0,0 +1,77 @@
+import os
+from datetime import datetime
+
+from dbt_platform_helper.providers.yaml_file import FileProvider
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+class CacheProvider:
+    def __init__(
+        self,
+        file_provider: FileProvider = None,
+    ):
+        self._cache_file = ".platform-helper-config-cache.yml"
+        self.file_provider = file_provider or YamlFileProvider
+
+    def read_supported_versions_from_cache(self, resource_name):
+
+        platform_helper_config = self.file_provider.load(self._cache_file)
+
+        return platform_helper_config.get(resource_name).get("versions")
+
+    def update_cache(self, resource_name, supported_versions):
+
+        platform_helper_config = {}
+
+        if self.__cache_exists():
+            platform_helper_config = self.file_provider.load(self._cache_file)
+
+        cache_dict = {
+            resource_name: {
+                "versions": supported_versions,
+                "date-retrieved": datetime.now().strftime("%d-%m-%y %H:%M:%S"),
+            }
+        }
+
+        platform_helper_config.update(cache_dict)
+
+        self.file_provider.write(
+            self._cache_file,
+            platform_helper_config,
+            "# [!] This file is autogenerated via the platform-helper. Do not edit.\n",
+        )
+
+    def cache_refresh_required(self, resource_name) -> bool:
+        """
+        Checks if the platform-helper should reach out to AWS to 'refresh' its
+        cached values.
+
+        An API call is needed if any of the following conditions are met:
+            1. No cache file (.platform-helper-config.yml) exists.
+            2. The resource name (e.g. redis, opensearch) does not exist within the cache file.
+            3. The date-retrieved value of the cached data is > than a time interval. In this case 1 day.
+        """
+
+        if not self.__cache_exists():
+            return True
+
+        platform_helper_config = self.file_provider.load(self._cache_file)
+
+        if platform_helper_config.get(resource_name):
+            return self.__check_if_cached_datetime_is_greater_than_interval(
+                platform_helper_config[resource_name].get("date-retrieved"), 1
+            )
+
+        return True
+
+    @staticmethod
+    def __check_if_cached_datetime_is_greater_than_interval(date_retrieved, interval_in_days):
+
+        current_datetime = datetime.now()
+        cached_datetime = datetime.strptime(date_retrieved, "%d-%m-%y %H:%M:%S")
+        delta = current_datetime - cached_datetime
+
+        return delta.days > interval_in_days
+
+    def __cache_exists(self):
+        return os.path.exists(self._cache_file)

dbt_platform_helper/providers/cloudformation.py

@@ -93,7 +93,6 @@ class CloudFormation:
         )
 
         params = []
-        # TODO Currently not covered by tests - see https://uktrade.atlassian.net/browse/DBTP-1582
         if "Parameters" in template_yml:
             for param in template_yml["Parameters"]:
                 params.append({"ParameterKey": param, "UsePreviousValue": True})