dbt-platform-helper 12.0.2__py3-none-any.whl → 12.2.0__py3-none-any.whl

dbt_platform_helper/domain/codebase.py

@@ -0,0 +1,222 @@
+import json
+import stat
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+import click
+import requests
+import yaml
+from boto3 import Session
+
+from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
+from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
+from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
+from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
+from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.aws import check_codebase_exists
+from dbt_platform_helper.utils.aws import check_image_exists
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.aws import get_build_url_from_arn
+from dbt_platform_helper.utils.aws import list_latest_images
+from dbt_platform_helper.utils.aws import start_build_extraction
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.git import check_if_commit_exists
+from dbt_platform_helper.utils.template import setup_templates
+
+
+class Codebase:
+    def __init__(
+        self,
+        input_fn: Callable[[str], str] = click.prompt,
+        echo_fn: Callable[[str], str] = click.secho,
+        confirm_fn: Callable[[str], bool] = click.confirm,
+        load_application_fn: Callable[[str], Application] = load_application,
+        get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
+        check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
+        check_image_exists_fn: Callable[[str], str] = check_image_exists,
+        get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
+        list_latest_images_fn: Callable[[str], str] = list_latest_images,
+        start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
+        check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
+        subprocess: Callable[[str], str] = subprocess.run,
+    ):
+        self.input_fn = input_fn
+        self.echo_fn = echo_fn
+        self.confirm_fn = confirm_fn
+        self.load_application_fn = load_application_fn
+        self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
+        self.check_codebase_exists_fn = check_codebase_exists_fn
+        self.check_image_exists_fn = check_image_exists_fn
+        self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
+        self.list_latest_images_fn = list_latest_images_fn
+        self.start_build_extraction_fn = start_build_extraction_fn
+        self.check_if_commit_exists_fn = check_if_commit_exists_fn
+        self.subprocess = subprocess
+
+    def prepare(self):
+        """Sets up an application codebase for use within a DBT platform
+        project."""
+        templates = setup_templates()
+
+        repository = (
+            self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
+            .stdout.split("/")[-1]
+            .strip()
+            .removesuffix(".git")
+        )
+        if repository.endswith("-deploy") or Path("./copilot").exists():
+            raise NotInCodeBaseRepositoryError
+
+        builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
+        builder_configuration_response = requests.get(builder_configuration_url)
+        builder_configuration_content = yaml.safe_load(
+            builder_configuration_response.content.decode("utf-8")
+        )
+        builder_versions = next(
+            (
+                item
+                for item in builder_configuration_content["builders"]
+                if item["name"] == "paketobuildpacks/builder-jammy-base"
+            ),
+            None,
+        )
+        builder_version = max(x["version"] for x in builder_versions["versions"])
+        builder_version = min(builder_version, "0.4.240")
+
+        Path("./.copilot/phases").mkdir(parents=True, exist_ok=True)
+        image_build_run_contents = templates.get_template(f".copilot/image_build_run.sh").render()
+
+        config_contents = templates.get_template(f".copilot/config.yml").render(
+            repository=repository, builder_version=builder_version
+        )
+        self.echo_fn(
+            mkfile(
+                Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
+            )
+        )
+
+        image_build_run_file = Path(".copilot/image_build_run.sh")
+        image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
+
+        self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
+
+        for phase in ["build", "install", "post_build", "pre_build"]:
+            phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
+
+            self.echo_fn(
+                mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
+            )
+
+    def build(self, app: str, codebase: str, commit: str):
+        """Trigger a CodePipeline pipeline based build."""
+        session = self.get_aws_session_or_abort_fn()
+        self.load_application_fn(app, default_session=session)
+
+        self.check_if_commit_exists_fn(commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
+            {
+                "projectName": f"codebuild-{app}-{codebase}",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceVersion": commit,
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def deploy(self, app, env, codebase, commit):
+        """Trigger a CodePipeline pipeline based deployment."""
+        session = self.get_aws_session_or_abort_fn()
+
+        application = self.load_application_fn(app, default_session=session)
+        if not application.environments.get(env):
+            raise ApplicationEnvironmentNotFoundError()
+
+        json.loads(self.check_codebase_exists_fn(session, application, codebase))
+
+        self.check_image_exists_fn(session, application, codebase, commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
+            {
+                "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceTypeOverride": "NO_SOURCE",
+                "environmentVariablesOverride": [
+                    {"name": "COPILOT_ENVIRONMENT", "value": env},
+                    {"name": "IMAGE_TAG", "value": f"commit-{commit}"},
+                ],
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                "Your deployment has been triggered. Check your build progress in the AWS Console: "
+                f"{build_url}",
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def list(self, app: str, with_images: bool):
+        """List available codebases for the application."""
+        session = self.get_aws_session_or_abort_fn()
+        application = self.load_application_fn(app, session)
+        ssm_client = session.client("ssm")
+        ecr_client = session.client("ecr")
+        codebases = self.__get_codebases(application, ssm_client)
+
+        self.echo_fn("The following codebases are available:")
+
+        for codebase in codebases:
+            self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
+            if with_images:
+                self.list_latest_images_fn(
+                    ecr_client,
+                    f"{application.name}/{codebase['name']}",
+                    codebase["repository"],
+                    self.echo_fn,
+                )
+
+        self.echo_fn("")
+
+    def __get_codebases(self, application, ssm_client):
+        parameters = ssm_client.get_parameters_by_path(
+            Path=f"/copilot/applications/{application.name}/codebases",
+            Recursive=True,
+        )["Parameters"]
+
+        codebases = [json.loads(p["Value"]) for p in parameters]
+
+        if not codebases:
+            # TODO Is this really an error? Or just no codebases so we could return an empty list?
+            raise NoCopilotCodebasesFoundError
+        return codebases
+
+    def __start_build_with_confirmation(
+        self,
+        confirm_fn,
+        codebuild_client,
+        get_build_url_from_arn_fn,
+        confirmation_message,
+        build_options,
+    ):
+        if confirm_fn(confirmation_message):
+            build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
+            return get_build_url_from_arn_fn(build_arn)
+        return None

dbt_platform_helper/domain/conduit.py

@@ -0,0 +1,172 @@
+import subprocess
+from collections.abc import Callable
+
+import click
+
+from dbt_platform_helper.exceptions import ECSAgentNotRunning
+from dbt_platform_helper.providers.cloudformation import (
+    add_stack_delete_policy_to_task_role,
+)
+from dbt_platform_helper.providers.cloudformation import update_conduit_stack_resources
+from dbt_platform_helper.providers.cloudformation import (
+    wait_for_cloudformation_to_reach_status,
+)
+from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
+from dbt_platform_helper.providers.copilot import create_addon_client_task
+from dbt_platform_helper.providers.copilot import create_postgres_admin_task
+from dbt_platform_helper.providers.ecs import ecs_exec_is_available
+from dbt_platform_helper.providers.ecs import get_cluster_arn
+from dbt_platform_helper.providers.ecs import get_ecs_task_arns
+from dbt_platform_helper.providers.ecs import get_or_create_task_name
+from dbt_platform_helper.providers.secrets import get_addon_type
+from dbt_platform_helper.providers.secrets import get_parameter_name
+from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.utils.messages import abort_with_error
+
+
+class Conduit:
+    def __init__(
+        self,
+        application: Application,
+        echo_fn: Callable[[str], str] = click.secho,
+        subprocess_fn: subprocess = subprocess,
+        get_ecs_task_arns_fn=get_ecs_task_arns,
+        connect_to_addon_client_task_fn=connect_to_addon_client_task,
+        create_addon_client_task_fn=create_addon_client_task,
+        create_postgres_admin_task_fn=create_postgres_admin_task,
+        get_addon_type_fn=get_addon_type,
+        ecs_exec_is_available_fn=ecs_exec_is_available,
+        get_cluster_arn_fn=get_cluster_arn,
+        get_parameter_name_fn=get_parameter_name,
+        get_or_create_task_name_fn=get_or_create_task_name,
+        add_stack_delete_policy_to_task_role_fn=add_stack_delete_policy_to_task_role,
+        update_conduit_stack_resources_fn=update_conduit_stack_resources,
+        wait_for_cloudformation_to_reach_status_fn=wait_for_cloudformation_to_reach_status,
+        abort_fn=abort_with_error,
+    ):
+
+        self.application = application
+        self.subprocess_fn = subprocess_fn
+        self.echo_fn = echo_fn
+        self.get_ecs_task_arns_fn = get_ecs_task_arns_fn
+        self.connect_to_addon_client_task_fn = connect_to_addon_client_task_fn
+        self.create_addon_client_task_fn = create_addon_client_task_fn
+        self.create_postgres_admin_task = create_postgres_admin_task_fn
+        self.get_addon_type_fn = get_addon_type_fn
+        self.ecs_exec_is_available_fn = ecs_exec_is_available_fn
+        self.get_cluster_arn_fn = get_cluster_arn_fn
+        self.get_parameter_name_fn = get_parameter_name_fn
+        self.get_or_create_task_name_fn = get_or_create_task_name_fn
+        self.add_stack_delete_policy_to_task_role_fn = add_stack_delete_policy_to_task_role_fn
+        self.update_conduit_stack_resources_fn = update_conduit_stack_resources_fn
+        self.wait_for_cloudformation_to_reach_status_fn = wait_for_cloudformation_to_reach_status_fn
+        self.abort_fn = abort_fn
+
+    def start(self, env: str, addon_name: str, access: str = "read"):
+        clients = self._initialise_clients(env)
+        addon_type, cluster_arn, parameter_name, task_name = self._get_addon_details(
+            env, addon_name, access
+        )
+
+        self.echo_fn(f"Checking if a conduit task is already running for {addon_type}")
+        task_arn = self.get_ecs_task_arns_fn(clients["ecs"], cluster_arn, task_name)
+        if not task_arn:
+            self.echo_fn("Creating conduit task")
+            self.create_addon_client_task_fn(
+                clients["iam"],
+                clients["ssm"],
+                clients["secrets_manager"],
+                self.subprocess_fn,
+                self.application,
+                env,
+                addon_type,
+                addon_name,
+                task_name,
+                access,
+            )
+
+            self.echo_fn("Updating conduit task")
+            self._update_stack_resources(
+                clients["cloudformation"],
+                clients["iam"],
+                clients["ssm"],
+                self.application.name,
+                env,
+                addon_type,
+                addon_name,
+                task_name,
+                parameter_name,
+                access,
+            )
+
+            task_arn = self.get_ecs_task_arns_fn(clients["ecs"], cluster_arn, task_name)
+
+        else:
+            self.echo_fn("Conduit task already running")
+
+        self.echo_fn(f"Checking if exec is available for conduit task...")
+
+        try:
+            self.ecs_exec_is_available_fn(clients["ecs"], cluster_arn, task_arn)
+        except ECSAgentNotRunning:
+            self.abort_fn('ECS exec agent never reached "RUNNING" status')
+
+        self.echo_fn("Connecting to conduit task")
+        self.connect_to_addon_client_task_fn(
+            clients["ecs"], self.subprocess_fn, self.application.name, env, cluster_arn, task_name
+        )
+
+    def _initialise_clients(self, env):
+        return {
+            "ecs": self.application.environments[env].session.client("ecs"),
+            "iam": self.application.environments[env].session.client("iam"),
+            "ssm": self.application.environments[env].session.client("ssm"),
+            "cloudformation": self.application.environments[env].session.client("cloudformation"),
+            "secrets_manager": self.application.environments[env].session.client("secretsmanager"),
+        }
+
+    def _get_addon_details(self, env, addon_name, access):
+        ssm_client = self.application.environments[env].session.client("ssm")
+        ecs_client = self.application.environments[env].session.client("ecs")
+
+        addon_type = self.get_addon_type_fn(ssm_client, self.application.name, env, addon_name)
+        cluster_arn = self.get_cluster_arn_fn(ecs_client, self.application.name, env)
+        parameter_name = self.get_parameter_name_fn(
+            self.application.name, env, addon_type, addon_name, access
+        )
+        task_name = self.get_or_create_task_name_fn(
+            ssm_client, self.application.name, env, addon_name, parameter_name
+        )
+
+        return addon_type, cluster_arn, parameter_name, task_name
+
+    def _update_stack_resources(
+        self,
+        cloudformation_client,
+        iam_client,
+        ssm_client,
+        app_name,
+        env,
+        addon_type,
+        addon_name,
+        task_name,
+        parameter_name,
+        access,
+    ):
+        self.add_stack_delete_policy_to_task_role_fn(cloudformation_client, iam_client, task_name)
+        stack_name = self.update_conduit_stack_resources_fn(
+            cloudformation_client,
+            iam_client,
+            ssm_client,
+            app_name,
+            env,
+            addon_type,
+            addon_name,
+            task_name,
+            parameter_name,
+            access,
+        )
+        self.echo_fn("Waiting for conduit task update to complete...")
+        self.wait_for_cloudformation_to_reach_status_fn(
+            cloudformation_client, "stack_update_complete", stack_name
+        )

dbt_platform_helper/domain/database_copy.py

@@ -8,9 +8,9 @@ from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
 from dbt_platform_helper.exceptions import AWSException
 from dbt_platform_helper.utils.application import Application
-from dbt_platform_helper.utils.application import ApplicationNotFoundError
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import Vpc
 from dbt_platform_helper.utils.aws import get_connection_string

dbt_platform_helper/exceptions.py

@@ -18,3 +18,64 @@ class IncompatibleMinorVersion(ValidationException):
         super().__init__()
         self.app_version = app_version
         self.check_version = check_version
+
+
+class NoClusterError(AWSException):
+    pass
+
+
+class CreateTaskTimeoutError(AWSException):
+    pass
+
+
+class ParameterNotFoundError(AWSException):
+    pass
+
+
+class AddonNotFoundError(AWSException):
+    pass
+
+
+class InvalidAddonTypeError(AWSException):
+    def __init__(self, addon_type):
+        self.addon_type = addon_type
+
+
+class AddonTypeMissingFromConfigError(AWSException):
+    pass
+
+
+class CopilotCodebaseNotFoundError(Exception):
+    pass
+
+
+class NotInCodeBaseRepositoryError(Exception):
+    pass
+
+
+class NoCopilotCodebasesFoundError(Exception):
+    pass
+
+
+class ImageNotFoundError(Exception):
+    pass
+
+
+class ApplicationDeploymentNotTriggered(Exception):
+    pass
+
+
+class ApplicationNotFoundError(Exception):
+    pass
+
+
+class ApplicationEnvironmentNotFoundError(Exception):
+    pass
+
+
+class SecretNotFoundError(AWSException):
+    pass
+
+
+class ECSAgentNotRunning(AWSException):
+    pass

dbt_platform_helper/providers/cloudformation.py

@@ -0,0 +1,105 @@
+import json
+
+from cfn_tools import dump_yaml
+from cfn_tools import load_yaml
+
+
+def add_stack_delete_policy_to_task_role(cloudformation_client, iam_client, task_name: str):
+
+    stack_name = f"task-{task_name}"
+    stack_resources = cloudformation_client.list_stack_resources(StackName=stack_name)[
+        "StackResourceSummaries"
+    ]
+
+    for resource in stack_resources:
+        if resource["LogicalResourceId"] == "DefaultTaskRole":
+            task_role_name = resource["PhysicalResourceId"]
+            iam_client.put_role_policy(
+                RoleName=task_role_name,
+                PolicyName="DeleteCloudFormationStack",
+                PolicyDocument=json.dumps(
+                    {
+                        "Version": "2012-10-17",
+                        "Statement": [
+                            {
+                                "Action": ["cloudformation:DeleteStack"],
+                                "Effect": "Allow",
+                                "Resource": f"arn:aws:cloudformation:*:*:stack/{stack_name}/*",
+                            },
+                        ],
+                    },
+                ),
+            )
+
+
+def update_conduit_stack_resources(
+    cloudformation_client,
+    iam_client,
+    ssm_client,
+    application_name: str,
+    env: str,
+    addon_type: str,
+    addon_name: str,
+    task_name: str,
+    parameter_name: str,
+    access: str,
+):
+
+    conduit_stack_name = f"task-{task_name}"
+    template = cloudformation_client.get_template(StackName=conduit_stack_name)
+    template_yml = load_yaml(template["TemplateBody"])
+    template_yml["Resources"]["LogGroup"]["DeletionPolicy"] = "Retain"
+    template_yml["Resources"]["TaskNameParameter"] = load_yaml(
+        f"""
+        Type: AWS::SSM::Parameter
+        Properties:
+          Name: {parameter_name}
+          Type: String
+          Value: {task_name}
+        """
+    )
+
+    log_filter_role_arn = iam_client.get_role(RoleName="CWLtoSubscriptionFilterRole")["Role"]["Arn"]
+
+    destination_log_group_arns = json.loads(
+        ssm_client.get_parameter(Name="/copilot/tools/central_log_groups")["Parameter"]["Value"]
+    )
+
+    destination_arn = destination_log_group_arns["dev"]
+    if env.lower() in ("prod", "production"):
+        destination_arn = destination_log_group_arns["prod"]
+
+    template_yml["Resources"]["SubscriptionFilter"] = load_yaml(
+        f"""
+        Type: AWS::Logs::SubscriptionFilter
+        DeletionPolicy: Retain
+        Properties:
+          RoleArn: {log_filter_role_arn}
+          LogGroupName: /copilot/{task_name}
+          FilterName: /copilot/conduit/{application_name}/{env}/{addon_type}/{addon_name}/{task_name.rsplit("-", 1)[1]}/{access}
+          FilterPattern: ''
+          DestinationArn: {destination_arn}
+        """
+    )
+
+    params = []
+    if "Parameters" in template_yml:
+        for param in template_yml["Parameters"]:
+            # TODO testing missed in codecov, update test to assert on method call below with params including ExistingParameter from cloudformation template.
+            params.append({"ParameterKey": param, "UsePreviousValue": True})
+
+    cloudformation_client.update_stack(
+        StackName=conduit_stack_name,
+        TemplateBody=dump_yaml(template_yml),
+        Parameters=params,
+        Capabilities=["CAPABILITY_IAM"],
+    )
+
+    return conduit_stack_name
+
+
+# TODO Catch errors and raise a more human friendly Exception is the CloudFormation stack goes into a "unhappy" state, e.g. ROLLBACK_IN_PROGRESS. Currently we get things like botocore.exceptions.WaiterError: Waiter StackUpdateComplete failed: Waiter encountered a terminal failure state: For expression "Stacks[].StackStatus" we matched expected path: "UPDATE_ROLLBACK_COMPLETE" at least once
+def wait_for_cloudformation_to_reach_status(cloudformation_client, stack_status, stack_name):
+
+    waiter = cloudformation_client.get_waiter(stack_status)
+    waiter.wait(StackName=stack_name, WaiterConfig={"Delay": 5, "MaxAttempts": 20})