dbt-platform-helper 12.4.1__tar.gz → 12.5.0__tar.gz

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 12.4.1
+Version: 12.5.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/dbt_platform_helper/COMMANDS.md

@@ -551,9 +551,6 @@ platform-helper pipeline generate
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
 
-    (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
-    - Generates the copilot manifest.yml for copilot/environments/<environment>
-
 ## Usage
 
 ```

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/dbt_platform_helper/commands/config.py

@@ -8,11 +8,11 @@ import botocore
 import click
 from prettytable import PrettyTable
 
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
 from dbt_platform_helper.providers.validation import ValidationException
 from dbt_platform_helper.utils import versioning
 from dbt_platform_helper.utils.click import ClickDocOptGroup
-from dbt_platform_helper.utils.validation import config_file_check
 from dbt_platform_helper.utils.versioning import get_platform_helper_versions
 
 yes = "\033[92m✔\033[0m"
@@ -98,7 +98,7 @@ def deployment():
 
     recommendations = {}
 
-    config_file_check()
+    ConfigProvider.config_file_check()
 
     for template_file in addons_templates:
         generated_with_version = maybe

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/dbt_platform_helper/commands/copilot.py

@@ -7,17 +7,22 @@ from pathlib import PosixPath
 
 import click
 import yaml
+from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.files import generate_override_files
 from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.messages import abort_with_error
+from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.validation import config_file_check
 from dbt_platform_helper.utils.validation import validate_addons
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
@@ -35,14 +40,6 @@ SERVICE_TYPES = [
     "Worker Service",
 ]
 
-ADDON_TEMPLATE_MAP = {
-    "s3": ["addons/svc/s3-policy.yml"],
-    "s3-policy": ["addons/svc/s3-policy.yml"],
-    "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
-    "subscription-filter": ["addons/svc/subscription-filter.yml"],
-    "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
-}
-
 
 def list_copilot_local_environments():
     return [
@@ -244,25 +241,42 @@ def _get_s3_kms_alias_arns(session, application_name, config):
     return arns
 
 
+def copilot_provider():
+    return CopilotTemplating()
+
+
 @copilot.command()
 def make_addons():
     """Generate addons CloudFormation for each environment."""
-    output_dir = Path(".").absolute()
-    config_file_check()
+    try:
+        config_provider = ConfigProvider(ConfigValidator())
+        _make_addons(config_provider)
+    except Exception as exc:
+        abort_with_error(exc)
+
+
+def _make_addons(config_provider: ConfigProvider):
+    config_provider.config_file_check()
+    try:
+        config = config_provider.load_and_validate_platform_config()
+    except SchemaError as ex:
+        click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
+        raise click.Abort
 
     templates = setup_templates()
-    config = _get_config()
+    extensions = _get_extensions()
     session = get_aws_session_or_abort()
 
     application_name = get_application_name()
 
     click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
 
+    output_dir = Path(".").absolute()
     env_path = Path(f"copilot/environments/")
     env_addons_path = env_path / "addons"
     env_overrides_path = env_path / "overrides"
 
-    _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path)
+    _cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
     _generate_env_overrides(output_dir)
 
     svc_names = list_copilot_local_services()
@@ -271,33 +285,33 @@ def make_addons():
         _generate_svc_overrides(base_path, templates, svc_name)
 
     services = []
-    for addon_name, addon_config in config.items():
-        print(f">>>>>>>>> {addon_name}")
-        addon_type = addon_config.pop("type")
-        environments = addon_config.pop("environments")
+    for ext_name, ext_data in extensions.items():
+        extension = {**ext_data}
+        addon_type = extension.pop("type")
+        environments = extension.pop("environments")
         environment_addon_config = {
             "addon_type": addon_type,
             "environments": environments,
-            "name": addon_config.get("name", None) or addon_name,
-            "prefix": camel_case(addon_name),
-            "secret_name": addon_name.upper().replace("-", "_"),
-            **addon_config,
+            "name": extension.get("name", None) or ext_name,
+            "prefix": camel_case(ext_name),
+            "secret_name": ext_name.upper().replace("-", "_"),
+            **extension,
         }
 
         services.append(environment_addon_config)
 
         service_addon_config = {
             "application_name": application_name,
-            "name": addon_config.get("name", None) or addon_name,
-            "prefix": camel_case(addon_name),
+            "name": extension.get("name", None) or ext_name,
+            "prefix": camel_case(ext_name),
             "environments": environments,
-            **addon_config,
+            **extension,
         }
 
         log_destination_arns = get_log_destination_arn()
 
         if addon_type in ["s3", "s3-policy"]:
-            if config[addon_name].get("serve_static_content"):
+            if extensions[ext_name].get("serve_static_content"):
                 continue
 
             s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
@@ -307,8 +321,8 @@ def make_addons():
                 )
 
         _generate_service_addons(
-            addon_config,
-            addon_name,
+            extension,
+            ext_name,
             addon_type,
             output_dir,
             service_addon_config,
@@ -316,10 +330,15 @@ def make_addons():
             log_destination_arns,
         )
 
+    environments = config_provider.apply_environment_defaults(config)["environments"]
+
+    provider = copilot_provider()
+    provider.generate_cross_account_s3_policies(environments, extensions)
+
     click.echo(templates.get_template("addon-instructions.txt").render(services=services))
 
 
-def _get_config():
+def _get_extensions():
     config = _validate_and_normalise_extensions_config(PACKAGE_DIR / "default-extensions.yml")
     project_config = _validate_and_normalise_extensions_config(PLATFORM_CONFIG_FILE, "extensions")
     config.update(project_config)

dbt_platform_helper-12.5.0/dbt_platform_helper/commands/environment.py

@@ -0,0 +1,86 @@
+import click
+from schema import SchemaError
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
+from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+AVAILABLE_TEMPLATES = ["default", "migration", "dmas-migration"]
+
+
+@click.group(cls=ClickDocOptGroup)
+def environment():
+    """Commands affecting environments."""
+    check_platform_helper_version_needs_update()
+
+
+@environment.command()
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+@click.option("--svc", type=str, required=True, multiple=True, default=["web"])
+@click.option(
+    "--template",
+    type=click.Choice(AVAILABLE_TEMPLATES),
+    default="default",
+    help="The maintenance page you wish to put up.",
+)
+@click.option("--vpc", type=str)
+def offline(app, env, svc, template, vpc):
+    """Take load-balanced web services offline with a maintenance page."""
+    try:
+        MaintenancePageProvider().activate(app, env, svc, template, vpc)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
+
+
+@environment.command()
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+def online(app, env):
+    """Remove a maintenance page from an environment."""
+    try:
+        MaintenancePageProvider().deactivate(app, env)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
+
+
+@environment.command()
+@click.option("--vpc-name", hidden=True)
+@click.option("--name", "-n", required=True)
+def generate(name, vpc_name):
+    if vpc_name:
+        click.secho(
+            f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
+            fg="red",
+        )
+        raise click.Abort
+    try:
+        config_provider = ConfigProvider(ConfigValidator())
+        CopilotEnvironment(config_provider).generate(name)
+    except SchemaError as ex:
+        click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
+        raise click.Abort
+
+
+@environment.command(help="Generate terraform manifest for the specified environment.")
+@click.option(
+    "--name", "-n", required=True, help="The name of the environment to generate a manifest for."
+)
+@click.option(
+    "--terraform-platform-modules-version",
+    help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
+)
+def generate_terraform(name, terraform_platform_modules_version):
+    config_provider = ConfigProvider(ConfigValidator())
+    TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/dbt_platform_helper/commands/pipeline.py

@@ -6,6 +6,8 @@ from shutil import rmtree
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.aws import get_account_details
 from dbt_platform_helper.utils.aws import get_codestar_connection_arn
@@ -16,7 +18,6 @@ from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.git import git_remote
 from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.validation import load_and_validate_platform_config
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
@@ -60,11 +61,9 @@ def generate(terraform_platform_modules_version, deploy_branch):
 
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
-
-    (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
-    - Generates the copilot manifest.yml for copilot/environments/<environment>
     """
-    pipeline_config = load_and_validate_platform_config()
+    config_provider = ConfigProvider(ConfigValidator())
+    pipeline_config = config_provider.load_and_validate_platform_config()
 
     has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
     has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config

{dbt_platform_helper-12.4.1 → dbt_platform_helper-12.5.0}/dbt_platform_helper/constants.py

@@ -16,3 +16,12 @@ CONDUIT_ADDON_TYPES = [
     "redis",
 ]
 CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"
+HYPHENATED_APPLICATION_NAME = "hyphenated-application-name"
+ALPHANUMERIC_ENVIRONMENT_NAME = "alphanumericenvironmentname123"
+ALPHANUMERIC_SERVICE_NAME = "alphanumericservicename123"
+COPILOT_IDENTIFIER = "c0PIlotiD3ntIF3r"
+CLUSTER_NAME_SUFFIX = f"Cluster-{COPILOT_IDENTIFIER}"
+SERVICE_NAME_SUFFIX = f"Service-{COPILOT_IDENTIFIER}"
+REFRESH_TOKEN_MESSAGE = (
+    "To refresh this SSO session run `aws sso login` with the corresponding profile"
+)

dbt_platform_helper-12.5.0/dbt_platform_helper/domain/config_validator.py

@@ -0,0 +1,242 @@
+from typing import Callable
+
+import boto3
+import click
+
+from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
+from dbt_platform_helper.constants import ENVIRONMENTS_KEY
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.providers.opensearch import OpensearchProvider
+from dbt_platform_helper.providers.redis import RedisProvider
+from dbt_platform_helper.utils.messages import abort_with_error
+
+
+class ConfigValidator:
+
+    def __init__(self, validations: Callable[[dict], None] = None):
+        self.validations = validations or [
+            self.validate_supported_redis_versions,
+            self.validate_supported_opensearch_versions,
+            self.validate_environment_pipelines,
+            self.validate_codebase_pipelines,
+            self.validate_environment_pipelines_triggers,
+            self.validate_database_copy_section,
+        ]
+
+    def run_validations(self, config: dict):
+        for validation in self.validations:
+            validation(config)
+
+    def _validate_extension_supported_versions(
+        self, config, extension_type, version_key, get_supported_versions
+    ):
+        extensions = config.get("extensions", {})
+        if not extensions:
+            return
+
+        extensions_for_type = [
+            extension
+            for extension in config.get("extensions", {}).values()
+            if extension.get("type") == extension_type
+        ]
+
+        supported_extension_versions = get_supported_versions()
+        extensions_with_invalid_version = []
+
+        for extension in extensions_for_type:
+
+            environments = extension.get("environments", {})
+
+            if not isinstance(environments, dict):
+                click.secho(
+                    f"Error: {extension_type} extension definition is invalid type, expected dictionary",
+                    fg="red",
+                )
+                continue
+            for environment, env_config in environments.items():
+
+                # An extension version doesn't need to be specified for all environments, provided one is specified under "*".
+                # So check if the version is set before checking if it's supported
+                extension_version = env_config.get(version_key)
+                if extension_version and extension_version not in supported_extension_versions:
+                    extensions_with_invalid_version.append(
+                        {"environment": environment, "version": extension_version}
+                    )
+
+        for version_failure in extensions_with_invalid_version:
+            click.secho(
+                f"{extension_type} version for environment {version_failure['environment']} is not in the list of supported {extension_type} versions: {supported_extension_versions}. Provided Version: {version_failure['version']}",
+                fg="red",
+            )
+
+    def validate_supported_redis_versions(self, config):
+        return self._validate_extension_supported_versions(
+            config=config,
+            extension_type="redis",
+            version_key="engine",
+            get_supported_versions=RedisProvider(
+                boto3.client("elasticache")
+            ).get_supported_redis_versions,
+        )
+
+    def validate_supported_opensearch_versions(self, config):
+        return self._validate_extension_supported_versions(
+            config=config,
+            extension_type="opensearch",
+            version_key="engine",
+            get_supported_versions=OpensearchProvider(
+                boto3.client("opensearch")
+            ).get_supported_opensearch_versions,
+        )
+
+    def validate_environment_pipelines(self, config):
+        bad_pipelines = {}
+        for pipeline_name, pipeline in config.get("environment_pipelines", {}).items():
+            bad_envs = []
+            pipeline_account = pipeline.get("account", None)
+            if pipeline_account:
+                for env in pipeline.get("environments", {}).keys():
+                    env_account = (
+                        config.get("environments", {})
+                        .get(env, {})
+                        .get("accounts", {})
+                        .get("deploy", {})
+                        .get("name")
+                    )
+                    if not env_account == pipeline_account:
+                        bad_envs.append(env)
+            if bad_envs:
+                bad_pipelines[pipeline_name] = {"account": pipeline_account, "bad_envs": bad_envs}
+        if bad_pipelines:
+            message = "The following pipelines are misconfigured:"
+            for pipeline, detail in bad_pipelines.items():
+                envs = detail["bad_envs"]
+                acc = detail["account"]
+                message += f"  '{pipeline}' - these environments are not in the '{acc}' account: {', '.join(envs)}\n"
+            abort_with_error(message)
+
+    def validate_codebase_pipelines(self, config):
+        if CODEBASE_PIPELINES_KEY in config:
+            for codebase in config[CODEBASE_PIPELINES_KEY]:
+                codebase_environments = []
+
+                for pipeline in codebase["pipelines"]:
+                    codebase_environments += [e["name"] for e in pipeline[ENVIRONMENTS_KEY]]
+
+                unique_codebase_environments = sorted(list(set(codebase_environments)))
+
+                if sorted(codebase_environments) != sorted(unique_codebase_environments):
+                    abort_with_error(
+                        f"The {PLATFORM_CONFIG_FILE} file is invalid, each environment can only be "
+                        "listed in a single pipeline per codebase"
+                    )
+
+    def validate_environment_pipelines_triggers(self, config):
+        errors = []
+        pipelines_with_triggers = {
+            pipeline_name: pipeline
+            for pipeline_name, pipeline in config.get("environment_pipelines", {}).items()
+            if "pipeline_to_trigger" in pipeline
+        }
+
+        for pipeline_name, pipeline in pipelines_with_triggers.items():
+            pipeline_to_trigger = pipeline["pipeline_to_trigger"]
+            if pipeline_to_trigger not in config.get("environment_pipelines", {}):
+                message = f"  '{pipeline_name}' - '{pipeline_to_trigger}' is not a valid target pipeline to trigger"
+
+                errors.append(message)
+                continue
+
+            if pipeline_to_trigger == pipeline_name:
+                message = f"  '{pipeline_name}' - pipelines cannot trigger themselves"
+                errors.append(message)
+
+        if errors:
+            error_message = "The following pipelines are misconfigured: \n"
+            abort_with_error(error_message + "\n  ".join(errors))
+
+    def validate_database_copy_section(self, config):
+        extensions = config.get("extensions", {})
+        if not extensions:
+            return
+
+        postgres_extensions = {
+            key: ext for key, ext in extensions.items() if ext.get("type", None) == "postgres"
+        }
+
+        if not postgres_extensions:
+            return
+
+        errors = []
+
+        for extension_name, extension in postgres_extensions.items():
+            database_copy_sections = extension.get("database_copy", [])
+
+            if not database_copy_sections:
+                return
+
+            all_environments = [
+                env for env in config.get("environments", {}).keys() if not env == "*"
+            ]
+            all_envs_string = ", ".join(all_environments)
+
+            for section in database_copy_sections:
+                from_env = section["from"]
+                to_env = section["to"]
+
+                from_account = (
+                    config.get("environments", {})
+                    .get(from_env, {})
+                    .get("accounts", {})
+                    .get("deploy", {})
+                    .get("id")
+                )
+                to_account = (
+                    config.get("environments", {})
+                    .get(to_env, {})
+                    .get("accounts", {})
+                    .get("deploy", {})
+                    .get("id")
+                )
+
+                if from_env == to_env:
+                    errors.append(
+                        f"database_copy 'to' and 'from' cannot be the same environment in extension '{extension_name}'."
+                    )
+
+                if "prod" in to_env:
+                    errors.append(
+                        f"Copying to a prod environment is not supported: database_copy 'to' cannot be '{to_env}' in extension '{extension_name}'."
+                    )
+
+                if from_env not in all_environments:
+                    errors.append(
+                        f"database_copy 'from' parameter must be a valid environment ({all_envs_string}) but was '{from_env}' in extension '{extension_name}'."
+                    )
+
+                if to_env not in all_environments:
+                    errors.append(
+                        f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
+                    )
+
+                if from_account != to_account:
+                    if "from_account" not in section:
+                        errors.append(
+                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'from_account' parameter must be present."
+                        )
+                    elif section["from_account"] != from_account:
+                        errors.append(
+                            f"Incorrect value for 'from_account' for environment '{from_env}'"
+                        )
+
+                    if "to_account" not in section:
+                        errors.append(
+                            f"Environments '{from_env}' and '{to_env}' are in different AWS accounts. The 'to_account' parameter must be present."
+                        )
+                    elif section["to_account"] != to_account:
+                        errors.append(
+                            f"Incorrect value for 'to_account' for environment '{to_env}'"
+                        )
+
+        if errors:
+            abort_with_error("\n".join(errors))