dbt-platform-helper 12.6.0__tar.gz → 13.0.0__tar.gz

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 12.6.0
+Version: 13.0.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/COMMANDS.md

@@ -222,7 +222,7 @@ platform-helper codebase build --app <application> --codebase <codebase>
 - `--app <text>`
   - AWS application name
 - `--codebase <text>`
-  - The codebase name as specified in the platform-config.yml file
+  - The codebase name as specified in the platform-config.yml file. This must be run from your codebase repository directory.
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._
@@ -246,7 +246,7 @@ platform-helper codebase deploy --app <application> --env <environment> --codeba
 - `--env <text>`
   - AWS Copilot environment
 - `--codebase <text>`
-  - The codebase name as specified in the platform-config.yml file
+  - The codebase name as specified in the platform-config.yml file. This can be run from any directory.
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._
@@ -492,7 +492,7 @@ platform-helper environment generate-terraform --name <name> [--terraform-platfo
 -n <text>`
   - The name of the environment to generate a manifest for.
 - `--terraform-platform-modules-version <text>`
-  - Override the default version of terraform-platform-modules. (Default version is '5').
+  - Override the default version of terraform-platform-modules. (Default version is '7').
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -565,7 +565,7 @@ platform-helper pipeline generate [--terraform-platform-modules-version <terrafo
   - Override the default version of terraform-platform-modules with a specific version or branch. 
 Precedence of version used is version supplied via CLI, then the version found in 
 platform-config.yml/default_versions/terraform-platform-modules. 
-In absence of these inputs, defaults to version '5'.
+In absence of these inputs, defaults to version '7'.
 - `--deploy-branch <text>`
   - Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource. 
 This is generated from the terraform/environments-pipeline/<aws_account>/main.tf file. 
@@ -667,7 +667,7 @@ platform-helper notify (environment-progress|add-comment)
 
 [↩ Parent](#platform-helper-notify)
 
-    Send environment progress notifications
+    Send environment progress notifications. This creates (or updates if --slack-ref is provided) the top level message to the channel.
 
 ## Usage
 
@@ -695,7 +695,7 @@ platform-helper notify environment-progress <slack_channel_id> <slack_token>
 - `--commit-sha <text>`
 
 - `--slack-ref <text>`
-  - Slack message reference
+  - Slack message reference of the message to update
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -703,7 +703,7 @@ platform-helper notify environment-progress <slack_channel_id> <slack_token>
 
 [↩ Parent](#platform-helper-notify)
 
-    Add comment to a notification
+    Add a comment to an existing Slack message
 
 ## Usage
 

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/commands/codebase.py

@@ -44,7 +44,7 @@ def list(app, with_images):
 @click.option("--app", help="AWS application name", required=True)
 @click.option(
     "--codebase",
-    help="The codebase name as specified in the platform-config.yml file",
+    help="The codebase name as specified in the platform-config.yml file. This must be run from your codebase repository directory.",
     required=True,
 )
 @click.option("--commit", help="GitHub commit hash", required=True)
@@ -61,7 +61,7 @@ def build(app, codebase, commit):
 @click.option("--env", help="AWS Copilot environment", required=True)
 @click.option(
     "--codebase",
-    help="The codebase name as specified in the platform-config.yml file",
+    help="The codebase name as specified in the platform-config.yml file. This can be run from any directory.",
     required=True,
 )
 @click.option("--commit", help="GitHub commit hash", required=True)

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/commands/notify.py

@@ -14,14 +14,16 @@ def notify():
     check_platform_helper_version_needs_update()
 
 
-@notify.command(help="Send environment progress notifications")
+@notify.command(
+    help="Send environment progress notifications. This creates (or updates if --slack-ref is provided) the top level message to the channel."
+)
 @click.argument("slack-channel-id")
 @click.argument("slack-token")
 @click.argument("message")
 @click.option("--build-arn")
 @click.option("--repository")
 @click.option("--commit-sha")
-@click.option("--slack-ref", help="Slack message reference")
+@click.option("--slack-ref", help="Slack message reference of the message to update")
 def environment_progress(
     slack_channel_id: str,
     slack_token: str,
@@ -83,7 +85,7 @@ def _get_slack_client(token: str):
     return WebClient(token=token)
 
 
-@notify.command(help="Add comment to a notification")
+@notify.command(help="Add a comment to an existing Slack message")
 @click.argument("slack-channel-id")
 @click.argument("slack-token")
 @click.argument("slack-ref")

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/commands/pipeline.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python
-
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.pipelines import Pipelines
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.ecr import ECRProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
 from dbt_platform_helper.utils.aws import get_codestar_connection_arn
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.git import git_remote
@@ -36,7 +38,7 @@ def pipeline():
     <application>-deploy/platform-config.yml/environment_pipelines/<environment-pipeline>/branch).""",
     default=None,
 )
-def generate(terraform_platform_modules_version, deploy_branch):
+def generate(terraform_platform_modules_version: str, deploy_branch: str):
     """
     Given a platform-config.yml file, generate environment and service
     deployment pipelines.
@@ -49,9 +51,16 @@ def generate(terraform_platform_modules_version, deploy_branch):
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
     """
-    pipelines = Pipelines(
-        ConfigProvider(ConfigValidator()),
-        git_remote,
-        get_codestar_connection_arn,
-    )
-    pipelines.generate(terraform_platform_modules_version, deploy_branch)
+    io = ClickIOProvider()
+    try:
+        pipelines = Pipelines(
+            ConfigProvider(ConfigValidator()),
+            TerraformManifestProvider(),
+            ECRProvider(),
+            git_remote,
+            get_codestar_connection_arn,
+            io,
+        )
+        pipelines.generate(terraform_platform_modules_version, deploy_branch)
+    except Exception as exc:
+        io.abort_with_error(str(exc))

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/constants.py

@@ -3,7 +3,9 @@ PLATFORM_CONFIG_FILE = "platform-config.yml"
 # Todo: Can we get rid of this yet?
 PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
 # Todo: Move to ???
-DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "7"
+SUPPORTED_TERRAFORM_VERSION = "~> 1.8"
+SUPPORTED_AWS_PROVIDER_VERSION = "~> 5"
 
 # Keys
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/domain/codebase.py

@@ -14,12 +14,13 @@ from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationException
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import check_codebase_exists
 from dbt_platform_helper.utils.aws import check_image_exists
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_build_url_from_arn
+from dbt_platform_helper.utils.aws import get_build_url_from_pipeline_execution_id
 from dbt_platform_helper.utils.aws import list_latest_images
 from dbt_platform_helper.utils.aws import start_build_extraction
+from dbt_platform_helper.utils.aws import start_pipeline_and_return_execution_id
 from dbt_platform_helper.utils.git import check_if_commit_exists
 from dbt_platform_helper.utils.template import setup_templates
 
@@ -30,22 +31,28 @@ class Codebase:
         io: ClickIOProvider = ClickIOProvider(),
         load_application: Callable[[str], Application] = load_application,
         get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
-        check_codebase_exists: Callable[[str], str] = check_codebase_exists,
         check_image_exists: Callable[[str], str] = check_image_exists,
         get_build_url_from_arn: Callable[[str], str] = get_build_url_from_arn,
+        get_build_url_from_pipeline_execution_id: Callable[
+            [str], str
+        ] = get_build_url_from_pipeline_execution_id,
         list_latest_images: Callable[[str], str] = list_latest_images,
         start_build_extraction: Callable[[str], str] = start_build_extraction,
+        start_pipeline_and_return_execution_id: Callable[
+            [str], str
+        ] = start_pipeline_and_return_execution_id,
         check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
         run_subprocess: Callable[[str], str] = subprocess.run,
     ):
         self.io = io
         self.load_application = load_application
         self.get_aws_session_or_abort = get_aws_session_or_abort
-        self.check_codebase_exists = check_codebase_exists
         self.check_image_exists = check_image_exists
         self.get_build_url_from_arn = get_build_url_from_arn
+        self.get_build_url_from_pipeline_execution_id = get_build_url_from_pipeline_execution_id
         self.list_latest_images = list_latest_images
         self.start_build_extraction = start_build_extraction
+        self.start_pipeline_and_return_execution_id = start_pipeline_and_return_execution_id
         self.check_if_commit_exists = check_if_commit_exists
         self.run_subprocess = run_subprocess
 
@@ -122,7 +129,7 @@ class Codebase:
             self.get_build_url_from_arn,
             f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
             {
-                "projectName": f"codebuild-{app}-{codebase}",
+                "projectName": f"{app}-{codebase}-codebase-pipeline-image-build",
                 "artifactsOverride": {"type": "NO_ARTIFACTS"},
                 "sourceVersion": commit,
             },
@@ -143,21 +150,19 @@ class Codebase:
         if not application.environments.get(env):
             raise ApplicationEnvironmentNotFoundException(env)
 
-        self.check_codebase_exists(session, application, codebase)
-
         self.check_image_exists(session, application, codebase, commit)
 
-        codebuild_client = session.client("codebuild")
-        build_url = self.__start_build_with_confirmation(
-            codebuild_client,
-            self.get_build_url_from_arn,
-            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
+        pipeline_name = f"{app}-{codebase}-manual-release-pipeline"
+        codepipeline_client = session.client("codepipeline")
+
+        build_url = self.__start_pipeline_execution_with_confirmation(
+            codepipeline_client,
+            self.get_build_url_from_pipeline_execution_id,
+            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment using the "{pipeline_name}" deployment pipeline. Do you want to continue?',
             {
-                "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
-                "artifactsOverride": {"type": "NO_ARTIFACTS"},
-                "sourceTypeOverride": "NO_SOURCE",
-                "environmentVariablesOverride": [
-                    {"name": "COPILOT_ENVIRONMENT", "value": env},
+                "name": pipeline_name,
+                "variables": [
+                    {"name": "ENVIRONMENT", "value": env},
                     {"name": "IMAGE_TAG", "value": f"commit-{commit}"},
                 ],
             },
@@ -217,6 +222,20 @@ class Codebase:
             return get_build_url_from_arn(build_arn)
         return None
 
+    def __start_pipeline_execution_with_confirmation(
+        self,
+        codepipeline_client,
+        get_build_url_from_pipeline_execution_id,
+        confirmation_message,
+        build_options,
+    ):
+        if self.io.confirm(confirmation_message):
+            execution_id = self.start_pipeline_and_return_execution_id(
+                codepipeline_client, build_options
+            )
+            return get_build_url_from_pipeline_execution_id(execution_id, build_options["name"])
+        return None
+
 
 class ApplicationDeploymentNotTriggered(PlatformException):
     def __init__(self, codebase: str):

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/domain/config_validator.py

@@ -2,13 +2,9 @@ from typing import Callable
 
 import boto3
 
-from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
-from dbt_platform_helper.constants import ENVIRONMENTS_KEY
-from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.opensearch import OpensearchProvider
 from dbt_platform_helper.providers.redis import RedisProvider
-from dbt_platform_helper.utils.messages import abort_with_error
 
 
 class ConfigValidator:
@@ -20,7 +16,6 @@ class ConfigValidator:
             self.validate_supported_redis_versions,
             self.validate_supported_opensearch_versions,
             self.validate_environment_pipelines,
-            self.validate_codebase_pipelines,
             self.validate_environment_pipelines_triggers,
             self.validate_database_copy_section,
             self.validate_database_migration_input_sources,
@@ -115,23 +110,7 @@ class ConfigValidator:
                 envs = detail["bad_envs"]
                 acc = detail["account"]
                 message += f"  '{pipeline}' - these environments are not in the '{acc}' account: {', '.join(envs)}\n"
-            abort_with_error(message)
-
-    def validate_codebase_pipelines(self, config):
-        if CODEBASE_PIPELINES_KEY in config:
-            for codebase in config[CODEBASE_PIPELINES_KEY]:
-                codebase_environments = []
-
-                for pipeline in codebase["pipelines"]:
-                    codebase_environments += [e["name"] for e in pipeline[ENVIRONMENTS_KEY]]
-
-                unique_codebase_environments = sorted(list(set(codebase_environments)))
-
-                if sorted(codebase_environments) != sorted(unique_codebase_environments):
-                    abort_with_error(
-                        f"The {PLATFORM_CONFIG_FILE} file is invalid, each environment can only be "
-                        "listed in a single pipeline per codebase"
-                    )
+            self.io.abort_with_error(message)
 
     def validate_environment_pipelines_triggers(self, config):
         errors = []
@@ -155,7 +134,7 @@ class ConfigValidator:
 
         if errors:
             error_message = "The following pipelines are misconfigured: \n"
-            abort_with_error(error_message + "\n  ".join(errors))
+            self.io.abort_with_error(error_message + "\n  ".join(errors))
 
     def validate_database_copy_section(self, config):
         extensions = config.get("extensions", {})
@@ -241,9 +220,9 @@ class ConfigValidator:
                         )
 
         if errors:
-            abort_with_error("\n".join(errors))
+            self.io.abort_with_error("\n".join(errors))
 
-    def validate_database_migration_input_sources(self, config):
+    def validate_database_migration_input_sources(self, config: dict):
         extensions = config.get("extensions", {})
         if not extensions:
             return
@@ -270,6 +249,5 @@ class ConfigValidator:
                     errors.append(
                         f"Error in '{extension_name}.environments.{env}.data_migration': 'import_sources' property is missing."
                     )
-
         if errors:
-            abort_with_error("\n".join(errors))
+            self.io.abort_with_error("\n".join(errors))

{dbt_platform_helper-12.6.0 → dbt_platform_helper-13.0.0}/dbt_platform_helper/domain/maintenance_page.py

@@ -2,6 +2,7 @@ import itertools
 import random
 import re
 import string
+import traceback
 from pathlib import Path
 from typing import Callable
 from typing import List
@@ -28,10 +29,32 @@ class MaintenancePageException(PlatformException):
 
 
 class LoadBalancedWebServiceNotFoundException(MaintenancePageException):
-    def __init__(self, application_name):
+    def __init__(self, application_name: str):
         super().__init__(f"No services deployed yet to {application_name} ")
 
 
+class FailedToActivateMaintenancePageException(MaintenancePageException):
+    def __init__(
+        self,
+        application_name: str,
+        env: str,
+        original_exception: Exception,
+        rolled_back_rules: dict[str, bool] = {},
+    ):
+        super().__init__(
+            f"Maintenance page failed to activate for the {application_name} application in environment {env}."
+        )
+        self.orginal_exception = original_exception
+        self.rolled_back_rules = rolled_back_rules
+
+    def __str__(self):
+        return (
+            f"{super().__str__()}\n"
+            f"Rolled-back rules: {self.rolled_back_rules }\n"
+            f"Original exception: {self.orginal_exception}"
+        )
+
+
 def get_maintenance_page_type(session: boto3.Session, listener_arn: str) -> Union[str, None]:
     lb_client = session.client("elbv2")
 
@@ -75,85 +98,99 @@ def add_maintenance_page(
     bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
 
     rule_priority = itertools.count(start=1)
+    try:
+        for svc in services:
+            target_group_arn = find_target_group(app, env, svc.name, session)
+
+            # not all of an application's services are guaranteed to have been deployed to an environment
+            if not target_group_arn:
+                continue
+
+            for ip in allowed_ips:
+                create_header_rule(
+                    lb_client,
+                    listener_arn,
+                    target_group_arn,
+                    "X-Forwarded-For",
+                    [ip],
+                    "AllowedIps",
+                    next(rule_priority),
+                )
+                create_source_ip_rule(
+                    lb_client,
+                    listener_arn,
+                    target_group_arn,
+                    [ip],
+                    "AllowedSourceIps",
+                    next(rule_priority),
+                )
 
-    for svc in services:
-        target_group_arn = find_target_group(app, env, svc.name, session)
-
-        # not all of an application's services are guaranteed to have been deployed to an environment
-        if not target_group_arn:
-            continue
-
-        for ip in allowed_ips:
             create_header_rule(
                 lb_client,
                 listener_arn,
                 target_group_arn,
-                "X-Forwarded-For",
-                [ip],
-                "AllowedIps",
+                "Bypass-Key",
+                [bypass_value],
+                "BypassIpFilter",
                 next(rule_priority),
             )
-            create_source_ip_rule(
-                lb_client,
-                listener_arn,
-                target_group_arn,
-                [ip],
-                "AllowedSourceIps",
-                next(rule_priority),
-            )
-
-        create_header_rule(
-            lb_client,
-            listener_arn,
-            target_group_arn,
-            "Bypass-Key",
-            [bypass_value],
-            "BypassIpFilter",
-            next(rule_priority),
-        )
 
         click.secho(
             f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
         )
 
-    lb_client.create_rule(
-        ListenerArn=listener_arn,
-        Priority=next(rule_priority),
-        Conditions=[
-            {
-                "Field": "path-pattern",
-                "PathPatternConfig": {"Values": ["/*"]},
-            }
-        ],
-        Actions=[
-            {
-                "Type": "fixed-response",
-                "FixedResponseConfig": {
-                    "StatusCode": "503",
-                    "ContentType": "text/html",
-                    "MessageBody": maintenance_page_content,
-                },
-            }
-        ],
-        Tags=[
-            {"Key": "name", "Value": "MaintenancePage"},
-            {"Key": "type", "Value": template},
-        ],
-    )
+        lb_client.create_rule(
+            ListenerArn=listener_arn,
+            Priority=next(rule_priority),
+            Conditions=[
+                {
+                    "Field": "path-pattern",
+                    "PathPatternConfig": {"Values": ["/*"]},
+                }
+            ],
+            Actions=[
+                {
+                    "Type": "fixed-response",
+                    "FixedResponseConfig": {
+                        "StatusCode": "503",
+                        "ContentType": "text/html",
+                        "MessageBody": maintenance_page_content,
+                    },
+                }
+            ],
+            Tags=[
+                {"Key": "name", "Value": "MaintenancePage"},
+                {"Key": "type", "Value": template},
+            ],
+        )
+    except Exception as e:
+        deleted_rules = clean_up_maintenance_page_rules(session, listener_arn)
+        raise FailedToActivateMaintenancePageException(
+            app, env, f"{e}:\n {traceback.format_exc()}", deleted_rules
+        )
 
 
-def remove_maintenance_page(session: boto3.Session, listener_arn: str):
+def clean_up_maintenance_page_rules(
+    session: boto3.Session, listener_arn: str, fail_when_not_deleted: bool = False
+) -> dict[str, bool]:
     lb_client = session.client("elbv2")
 
     rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
     tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
 
+    deletes = {}
     for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
         deleted = delete_listener_rule(tag_descriptions, name, lb_client)
-
-        if name == "MaintenancePage" and not deleted:
+        deletes[name] = bool(deleted)
+        if fail_when_not_deleted and name == "MaintenancePage" and not deleted:
             raise ListenerRuleNotFoundException()
 
+    return deletes
+
+
+def remove_maintenance_page(session: boto3.Session, listener_arn: str):
+    clean_up_maintenance_page_rules(session, listener_arn, True)
+
 
 class MaintenancePage:
     def __init__(