dbt-platform-helper 12.2.4__tar.gz → 12.3.0__tar.gz

{dbt_platform_helper-12.2.4 → dbt_platform_helper-12.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 12.2.4
+Version: 12.3.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

dbt_platform_helper-12.3.0/dbt_platform_helper/commands/codebase.py

@@ -0,0 +1,75 @@
+import click
+
+from dbt_platform_helper.domain.codebase import Codebase
+from dbt_platform_helper.exceptions import PlatformException
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def codebase():
+    """Codebase commands."""
+    check_platform_helper_version_needs_update()
+
+
+@codebase.command()
+def prepare():
+    """Sets up an application codebase for use within a DBT platform project."""
+    try:
+        Codebase().prepare()
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--with-images",
+    help="List up to the last 10 images tagged for this codebase",
+    default=False,
+    is_flag=True,
+)
+def list(app, with_images):
+    """List available codebases for the application."""
+    try:
+        Codebase().list(app, with_images)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def build(app, codebase, commit):
+    """Trigger a CodePipeline pipeline based build."""
+    try:
+        Codebase().build(app, codebase, commit)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option("--env", help="AWS Copilot environment", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def deploy(app, env, codebase, commit):
+    try:
+        Codebase().deploy(app, env, codebase, commit)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort

dbt_platform_helper-12.3.0/dbt_platform_helper/commands/conduit.py

@@ -0,0 +1,58 @@
+import click
+
+from dbt_platform_helper.domain.conduit import Conduit
+from dbt_platform_helper.exceptions import AWSException
+from dbt_platform_helper.providers.cloudformation import CloudFormation
+from dbt_platform_helper.providers.ecs import ECS
+from dbt_platform_helper.providers.secrets import Secrets
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.click import ClickDocOptCommand
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
+
+
+@click.command(cls=ClickDocOptCommand)
+@click.argument("addon_name", type=str, required=True)
+@click.option("--app", help="Application name", required=True)
+@click.option("--env", help="Environment name", required=True)
+@click.option(
+    "--access",
+    default="read",
+    type=click.Choice(CONDUIT_ACCESS_OPTIONS),
+    help="Allow read, write or admin access to the database addons.",
+)
+def conduit(addon_name: str, app: str, env: str, access: str):
+    """Opens a shell for a given addon_name create a conduit connection to
+    interact with postgres, opensearch or redis."""
+    check_platform_helper_version_needs_update()
+    application = load_application(app)
+
+    try:
+        secrets_provider: Secrets = Secrets(
+            application.environments[env].session.client("ssm"),
+            application.environments[env].session.client("secretsmanager"),
+            application.name,
+            env,
+        )
+        cloudformation_provider: CloudFormation = CloudFormation(
+            application.environments[env].session.client("cloudformation"),
+            application.environments[env].session.client("iam"),
+            application.environments[env].session.client("ssm"),
+        )
+
+        ecs_provider: ECS = ECS(
+            application.environments[env].session.client("ecs"),
+            application.environments[env].session.client("ssm"),
+            application.name,
+            env,
+        )
+
+        Conduit(application, secrets_provider, cloudformation_provider, ecs_provider).start(
+            env, addon_name, access
+        )
+    except AWSException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort

{dbt_platform_helper-12.2.4 → dbt_platform_helper-12.3.0}/dbt_platform_helper/commands/secrets.py

@@ -102,7 +102,6 @@ def list(app, env):
     params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
     secrets = []
 
-    # TODO: refactor shared code with get_ssm_secret_names - Check if this is still valid
     while True:
         response = client.get_parameters_by_path(**params)
 
@@ -114,6 +113,7 @@ def list(app, env):
         else:
             break
 
+    # Todo: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
     print("\n".join(sorted(secrets)))
 
 

{dbt_platform_helper-12.2.4 → dbt_platform_helper-12.3.0}/dbt_platform_helper/domain/codebase.py

@@ -11,7 +11,6 @@ from boto3 import Session
 
 from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
 from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
-from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
 from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import load_application
@@ -29,31 +28,31 @@ from dbt_platform_helper.utils.template import setup_templates
 class Codebase:
     def __init__(
         self,
-        input_fn: Callable[[str], str] = click.prompt,
-        echo_fn: Callable[[str], str] = click.secho,
-        confirm_fn: Callable[[str], bool] = click.confirm,
-        load_application_fn: Callable[[str], Application] = load_application,
-        get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
-        check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
-        check_image_exists_fn: Callable[[str], str] = check_image_exists,
-        get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
-        list_latest_images_fn: Callable[[str], str] = list_latest_images,
-        start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
-        check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
-        subprocess: Callable[[str], str] = subprocess.run,
+        input: Callable[[str], str] = click.prompt,
+        echo: Callable[[str], str] = click.secho,
+        confirm: Callable[[str], bool] = click.confirm,
+        load_application: Callable[[str], Application] = load_application,
+        get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
+        check_codebase_exists: Callable[[str], str] = check_codebase_exists,
+        check_image_exists: Callable[[str], str] = check_image_exists,
+        get_build_url_from_arn: Callable[[str], str] = get_build_url_from_arn,
+        list_latest_images: Callable[[str], str] = list_latest_images,
+        start_build_extraction: Callable[[str], str] = start_build_extraction,
+        check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
+        run_subprocess: Callable[[str], str] = subprocess.run,
     ):
-        self.input_fn = input_fn
-        self.echo_fn = echo_fn
-        self.confirm_fn = confirm_fn
-        self.load_application_fn = load_application_fn
-        self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
-        self.check_codebase_exists_fn = check_codebase_exists_fn
-        self.check_image_exists_fn = check_image_exists_fn
-        self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
-        self.list_latest_images_fn = list_latest_images_fn
-        self.start_build_extraction_fn = start_build_extraction_fn
-        self.check_if_commit_exists_fn = check_if_commit_exists_fn
-        self.subprocess = subprocess
+        self.input = input
+        self.echo = echo
+        self.confirm = confirm
+        self.load_application = load_application
+        self.get_aws_session_or_abort = get_aws_session_or_abort
+        self.check_codebase_exists = check_codebase_exists
+        self.check_image_exists = check_image_exists
+        self.get_build_url_from_arn = get_build_url_from_arn
+        self.list_latest_images = list_latest_images
+        self.start_build_extraction = start_build_extraction
+        self.check_if_commit_exists = check_if_commit_exists
+        self.run_subprocess = run_subprocess
 
     def prepare(self):
         """Sets up an application codebase for use within a DBT platform
@@ -61,13 +60,15 @@ class Codebase:
         templates = setup_templates()
 
         repository = (
-            self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
+            self.run_subprocess(
+                ["git", "remote", "get-url", "origin"], capture_output=True, text=True
+            )
             .stdout.split("/")[-1]
             .strip()
             .removesuffix(".git")
         )
         if repository.endswith("-deploy") or Path("./copilot").exists():
-            raise NotInCodeBaseRepositoryError
+            raise NotInCodeBaseRepositoryError()
 
         builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
         builder_configuration_response = requests.get(builder_configuration_url)
@@ -91,7 +92,7 @@ class Codebase:
         config_contents = templates.get_template(f".copilot/config.yml").render(
             repository=repository, builder_version=builder_version
         )
-        self.echo_fn(
+        self.echo(
             mkfile(
                 Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
             )
@@ -100,27 +101,27 @@ class Codebase:
         image_build_run_file = Path(".copilot/image_build_run.sh")
         image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
 
-        self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
+        self.echo(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
 
         for phase in ["build", "install", "post_build", "pre_build"]:
             phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
 
-            self.echo_fn(
+            self.echo(
                 mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
             )
 
     def build(self, app: str, codebase: str, commit: str):
         """Trigger a CodePipeline pipeline based build."""
-        session = self.get_aws_session_or_abort_fn()
-        self.load_application_fn(app, default_session=session)
+        session = self.get_aws_session_or_abort()
+        self.load_application(app, default_session=session)
 
-        self.check_if_commit_exists_fn(commit)
+        self.check_if_commit_exists(commit)
 
         codebuild_client = session.client("codebuild")
         build_url = self.__start_build_with_confirmation(
-            self.confirm_fn,
+            self.confirm,
             codebuild_client,
-            self.get_build_url_from_arn_fn,
+            self.get_build_url_from_arn,
             f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
             {
                 "projectName": f"codebuild-{app}-{codebase}",
@@ -130,29 +131,29 @@ class Codebase:
         )
 
         if build_url:
-            return self.echo_fn(
+            return self.echo(
                 f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
             )
 
-        raise ApplicationDeploymentNotTriggered()
+        raise ApplicationDeploymentNotTriggered(codebase)
 
     def deploy(self, app, env, codebase, commit):
         """Trigger a CodePipeline pipeline based deployment."""
-        session = self.get_aws_session_or_abort_fn()
+        session = self.get_aws_session_or_abort()
 
-        application = self.load_application_fn(app, default_session=session)
+        application = self.load_application(app, default_session=session)
         if not application.environments.get(env):
-            raise ApplicationEnvironmentNotFoundError()
+            raise ApplicationEnvironmentNotFoundError(env)
 
-        self.check_codebase_exists_fn(session, application, codebase)
+        self.check_codebase_exists(session, application, codebase)
 
-        self.check_image_exists_fn(session, application, codebase, commit)
+        self.check_image_exists(session, application, codebase, commit)
 
         codebuild_client = session.client("codebuild")
         build_url = self.__start_build_with_confirmation(
-            self.confirm_fn,
+            self.confirm,
             codebuild_client,
-            self.get_build_url_from_arn_fn,
+            self.get_build_url_from_arn,
             f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
             {
                 "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
@@ -166,34 +167,34 @@ class Codebase:
         )
 
         if build_url:
-            return self.echo_fn(
+            return self.echo(
                 "Your deployment has been triggered. Check your build progress in the AWS Console: "
                 f"{build_url}",
             )
 
-        raise ApplicationDeploymentNotTriggered()
+        raise ApplicationDeploymentNotTriggered(codebase)
 
     def list(self, app: str, with_images: bool):
         """List available codebases for the application."""
-        session = self.get_aws_session_or_abort_fn()
-        application = self.load_application_fn(app, session)
+        session = self.get_aws_session_or_abort()
+        application = self.load_application(app, session)
         ssm_client = session.client("ssm")
         ecr_client = session.client("ecr")
         codebases = self.__get_codebases(application, ssm_client)
 
-        self.echo_fn("The following codebases are available:")
+        self.echo("The following codebases are available:")
 
         for codebase in codebases:
-            self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
+            self.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
             if with_images:
-                self.list_latest_images_fn(
+                self.list_latest_images(
                     ecr_client,
                     f"{application.name}/{codebase['name']}",
                     codebase["repository"],
-                    self.echo_fn,
+                    self.echo,
                 )
 
-        self.echo_fn("")
+        self.echo("")
 
     def __get_codebases(self, application, ssm_client):
         parameters = ssm_client.get_parameters_by_path(
@@ -204,19 +205,18 @@ class Codebase:
         codebases = [json.loads(p["Value"]) for p in parameters]
 
         if not codebases:
-            # TODO Is this really an error? Or just no codebases so we could return an empty list?
-            raise NoCopilotCodebasesFoundError
+            return []
         return codebases
 
     def __start_build_with_confirmation(
         self,
-        confirm_fn,
+        confirm,
         codebuild_client,
-        get_build_url_from_arn_fn,
+        get_build_url_from_arn,
         confirmation_message,
         build_options,
     ):
-        if confirm_fn(confirmation_message):
-            build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
-            return get_build_url_from_arn_fn(build_arn)
+        if confirm(confirmation_message):
+            build_arn = self.start_build_extraction(codebuild_client, build_options)
+            return get_build_url_from_arn(build_arn)
         return None

dbt_platform_helper-12.3.0/dbt_platform_helper/domain/conduit.py

@@ -0,0 +1,127 @@
+import subprocess
+from collections.abc import Callable
+
+import click
+
+from dbt_platform_helper.providers.cloudformation import CloudFormation
+from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
+from dbt_platform_helper.providers.copilot import create_addon_client_task
+from dbt_platform_helper.providers.copilot import create_postgres_admin_task
+from dbt_platform_helper.providers.ecs import ECS
+from dbt_platform_helper.providers.secrets import Secrets
+from dbt_platform_helper.utils.application import Application
+
+
+class Conduit:
+    def __init__(
+        self,
+        application: Application,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        ecs_provider: ECS,
+        echo: Callable[[str], str] = click.secho,
+        subprocess: subprocess = subprocess,
+        connect_to_addon_client_task=connect_to_addon_client_task,
+        create_addon_client_task=create_addon_client_task,
+        create_postgres_admin_task=create_postgres_admin_task,
+    ):
+
+        self.application = application
+        self.secrets_provider = secrets_provider
+        self.cloudformation_provider = cloudformation_provider
+        self.ecs_provider = ecs_provider
+        self.subprocess = subprocess
+        self.echo = echo
+        self.connect_to_addon_client_task = connect_to_addon_client_task
+        self.create_addon_client_task = create_addon_client_task
+        self.create_postgres_admin_task = create_postgres_admin_task
+
+    def start(self, env: str, addon_name: str, access: str = "read"):
+        clients = self._initialise_clients(env)
+        addon_type, cluster_arn, parameter_name, task_name = self._get_addon_details(
+            addon_name, access
+        )
+
+        self.echo(f"Checking if a conduit task is already running for {addon_type}")
+        task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
+        if not task_arns:
+            self.echo("Creating conduit task")
+            self.create_addon_client_task(
+                clients["iam"],
+                clients["ssm"],
+                clients["secrets_manager"],
+                self.subprocess,
+                self.application,
+                env,
+                addon_type,
+                addon_name,
+                task_name,
+                access,
+            )
+
+            self.echo("Updating conduit task")
+            self._update_stack_resources(
+                self.application.name,
+                env,
+                addon_type,
+                addon_name,
+                task_name,
+                parameter_name,
+                access,
+            )
+
+            task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
+
+        else:
+            self.echo("Conduit task already running")
+
+        self.echo(f"Checking if exec is available for conduit task...")
+
+        self.ecs_provider.ecs_exec_is_available(cluster_arn, task_arns)
+
+        self.echo("Connecting to conduit task")
+        self.connect_to_addon_client_task(
+            clients["ecs"], self.subprocess, self.application.name, env, cluster_arn, task_name
+        )
+
+    def _initialise_clients(self, env):
+        return {
+            "ecs": self.application.environments[env].session.client("ecs"),
+            "iam": self.application.environments[env].session.client("iam"),
+            "ssm": self.application.environments[env].session.client("ssm"),
+            "cloudformation": self.application.environments[env].session.client("cloudformation"),
+            "secrets_manager": self.application.environments[env].session.client("secretsmanager"),
+        }
+
+    def _get_addon_details(self, addon_name, access):
+        addon_type = self.secrets_provider.get_addon_type(addon_name)
+        cluster_arn = self.ecs_provider.get_cluster_arn()
+        parameter_name = self.secrets_provider.get_parameter_name(addon_type, addon_name, access)
+        task_name = self.ecs_provider.get_or_create_task_name(addon_name, parameter_name)
+
+        return addon_type, cluster_arn, parameter_name, task_name
+
+    def _update_stack_resources(
+        self,
+        app_name,
+        env,
+        addon_type,
+        addon_name,
+        task_name,
+        parameter_name,
+        access,
+    ):
+        self.cloudformation_provider.add_stack_delete_policy_to_task_role(task_name)
+        stack_name = self.cloudformation_provider.update_conduit_stack_resources(
+            app_name,
+            env,
+            addon_type,
+            addon_name,
+            task_name,
+            parameter_name,
+            access,
+        )
+        self.echo("Waiting for conduit task update to complete...")
+        self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
+            "stack_update_complete", stack_name
+        )