dbt-platform-helper 12.0.2__tar.gz → 12.2.0__tar.gz

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 12.0.2
+Version: 12.2.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team
@@ -12,6 +12,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: Jinja2 (==3.1.4)
 Requires-Dist: PyYAML (==6.0.1)
 Requires-Dist: aiohttp (>=3.8.4,<4.0.0)

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.2.0}/dbt_platform_helper/COMMANDS.md

@@ -222,7 +222,7 @@ platform-helper codebase build --app <application> --codebase <codebase>
 - `--app <text>`
   - AWS application name
 - `--codebase <text>`
-  - The codebase name as specified in the pipelines.yml file
+  - The codebase name as specified in the platform-config.yml file
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._
@@ -232,8 +232,6 @@ platform-helper codebase build --app <application> --codebase <codebase>
 
 [↩ Parent](#platform-helper-codebase)
 
-    Trigger a CodePipeline pipeline based deployment.
-
 ## Usage
 
 ```
@@ -248,7 +246,7 @@ platform-helper codebase deploy --app <application> --env <environment> --codeba
 - `--env <text>`
   - AWS Copilot environment
 - `--codebase <text>`
-  - The codebase name as specified in the pipelines.yml file
+  - The codebase name as specified in the platform-config.yml file
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._
@@ -258,7 +256,8 @@ platform-helper codebase deploy --app <application> --env <environment> --codeba
 
 [↩ Parent](#platform-helper)
 
-    Create a conduit connection to an addon.
+    Opens a shell for a given addon_name create a conduit connection to
+    interact with postgres, opensearch or redis.
 
 ## Usage
 
@@ -274,11 +273,11 @@ platform-helper conduit <addon_name>
 ## Options
 
 - `--app <text>`
-  - AWS application name
+  - Application name
 - `--env <text>`
-  - AWS environment name
+  - Environment name
 - `--access <choice>` _Defaults to read._
-  - Allow write or admin access to database addons
+  - Allow read, write or admin access to the database addons.
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.2.0}/dbt_platform_helper/commands/application.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 # application commands are deprecated, do not spend time refactoring them
+# Service teams are trained to use them as a replacement for cf app(s)
 
 import time
 from datetime import datetime

dbt_platform_helper-12.2.0/dbt_platform_helper/commands/codebase.py

@@ -0,0 +1,146 @@
+import json
+import os
+
+import click
+
+from dbt_platform_helper.domain.codebase import Codebase
+from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
+from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
+from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
+from dbt_platform_helper.exceptions import ImageNotFoundError
+from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
+from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.git import CommitNotFoundError
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def codebase():
+    """Codebase commands."""
+    check_platform_helper_version_needs_update()
+
+
+@codebase.command()
+def prepare():
+    """Sets up an application codebase for use within a DBT platform project."""
+    try:
+        Codebase().prepare()
+    except NotInCodeBaseRepositoryError:
+        # TODO: Set exception message in the exceptions and just output the message in the command code
+        click.secho(
+            "You are in the deploy repository; make sure you are in the application codebase repository.",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--with-images",
+    help="List up to the last 10 images tagged for this codebase",
+    default=False,
+    is_flag=True,
+)
+def list(app, with_images):
+    """List available codebases for the application."""
+    try:
+        Codebase().list(app, with_images)
+    except NoCopilotCodebasesFoundError:
+        click.secho(
+            f"""No codebases found for application "{app}""",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationNotFoundError:
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def build(app, codebase, commit):
+    """Trigger a CodePipeline pipeline based build."""
+    try:
+        Codebase().build(app, codebase, commit)
+    except ApplicationNotFoundError:
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+    except CommitNotFoundError:
+        click.secho(
+            f'The commit hash "{commit}" either does not exist or you need to run `git fetch`.',
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationDeploymentNotTriggered:
+        click.secho(
+            f"Your build for {codebase} was not triggered.",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option("--env", help="AWS Copilot environment", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def deploy(app, env, codebase, commit):
+    try:
+        Codebase().deploy(app, env, codebase, commit)
+    except ApplicationNotFoundError:
+        # TODO: Set exception message in the exceptions and just output the message in the command code
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationEnvironmentNotFoundError:
+        click.secho(
+            f"""The environment "{env}" either does not exist or has not been deployed.""",
+            fg="red",
+        )
+        raise click.Abort
+    except (
+        CopilotCodebaseNotFoundError,
+        # TODO: Catch this error earlier and throw a more meaningful error, maybe it's CopilotCodebaseNotFoundError?
+        json.JSONDecodeError,
+    ):
+        click.secho(
+            f"""The codebase "{codebase}" either does not exist or has not been deployed.""",
+            fg="red",
+        )
+        raise click.Abort
+    except ImageNotFoundError:
+        click.secho(
+            f'The commit hash "{commit}" has not been built into an image, try the '
+            "`platform-helper codebase build` command first.",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationDeploymentNotTriggered:
+        click.secho(
+            f"Your deployment for {codebase} was not triggered.",
+            fg="red",
+        )
+        raise click.Abort

dbt_platform_helper-12.2.0/dbt_platform_helper/commands/conduit.py

@@ -0,0 +1,78 @@
+import click
+
+from dbt_platform_helper.constants import CONDUIT_ADDON_TYPES
+from dbt_platform_helper.domain.conduit import Conduit
+from dbt_platform_helper.exceptions import AddonNotFoundError
+from dbt_platform_helper.exceptions import AddonTypeMissingFromConfigError
+from dbt_platform_helper.exceptions import CreateTaskTimeoutError
+from dbt_platform_helper.exceptions import InvalidAddonTypeError
+from dbt_platform_helper.exceptions import NoClusterError
+from dbt_platform_helper.exceptions import ParameterNotFoundError
+from dbt_platform_helper.providers.secrets import SecretNotFoundError
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.click import ClickDocOptCommand
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
+
+
+@click.command(cls=ClickDocOptCommand)
+@click.argument("addon_name", type=str, required=True)
+@click.option("--app", help="Application name", required=True)
+@click.option("--env", help="Environment name", required=True)
+@click.option(
+    "--access",
+    default="read",
+    type=click.Choice(CONDUIT_ACCESS_OPTIONS),
+    help="Allow read, write or admin access to the database addons.",
+)
+def conduit(addon_name: str, app: str, env: str, access: str):
+    """Opens a shell for a given addon_name create a conduit connection to
+    interact with postgres, opensearch or redis."""
+    check_platform_helper_version_needs_update()
+    application = load_application(app)
+
+    try:
+        Conduit(application).start(env, addon_name, access)
+    except NoClusterError:
+        # TODO: Set exception message in the exceptions and just output the message in the command code, should be able to catch all errors in one block
+        click.secho(f"""No ECS cluster found for "{app}" in "{env}" environment.""", fg="red")
+        exit(1)
+    except SecretNotFoundError as err:
+        click.secho(
+            f"""No secret called "{err}" for "{app}" in "{env}" environment.""",
+            fg="red",
+        )
+        exit(1)
+    except CreateTaskTimeoutError:
+        click.secho(
+            f"""Client ({addon_name}) ECS task has failed to start for "{app}" in "{env}" environment.""",
+            fg="red",
+        )
+        exit(1)
+    except ParameterNotFoundError:
+        click.secho(
+            f"""No parameter called "/copilot/applications/{app}/environments/{env}/addons". Try deploying the "{app}" "{env}" environment.""",
+            fg="red",
+        )
+        exit(1)
+    except AddonNotFoundError:
+        click.secho(
+            f"""Addon "{addon_name}" does not exist.""",
+            fg="red",
+        )
+        exit(1)
+    except InvalidAddonTypeError as err:
+        click.secho(
+            f"""Addon type "{err.addon_type}" is not supported, we support: {", ".join(CONDUIT_ADDON_TYPES)}.""",
+            fg="red",
+        )
+        exit(1)
+    except AddonTypeMissingFromConfigError:
+        click.secho(
+            f"""The configuration for the addon {addon_name}, is missconfigured and missing the addon type.""",
+            fg="red",
+        )
+        exit(1)

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.2.0}/dbt_platform_helper/commands/secrets.py

@@ -102,7 +102,7 @@ def list(app, env):
     params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
     secrets = []
 
-    # TODO: refactor shared code with get_ssm_secret_names
+    # TODO: refactor shared code with get_ssm_secret_names - Check if this is still valid
     while True:
         response = client.get_parameters_by_path(**params)
 

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.2.0}/dbt_platform_helper/constants.py

@@ -1,5 +1,16 @@
 PLATFORM_CONFIG_FILE = "platform-config.yml"
 PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
+DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+PLATFORM_HELPER_CACHE_FILE = ".platform-helper-config-cache.yml"
+
+# Keys
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"
 ENVIRONMENTS_KEY = "environments"
-DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+
+# Conduit
+CONDUIT_ADDON_TYPES = [
+    "opensearch",
+    "postgres",
+    "redis",
+]
+CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"

dbt_platform_helper-12.2.0/dbt_platform_helper/domain/codebase.py

@@ -0,0 +1,222 @@
+import json
+import stat
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+import click
+import requests
+import yaml
+from boto3 import Session
+
+from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
+from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
+from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
+from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
+from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.aws import check_codebase_exists
+from dbt_platform_helper.utils.aws import check_image_exists
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.aws import get_build_url_from_arn
+from dbt_platform_helper.utils.aws import list_latest_images
+from dbt_platform_helper.utils.aws import start_build_extraction
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.git import check_if_commit_exists
+from dbt_platform_helper.utils.template import setup_templates
+
+
+class Codebase:
+    def __init__(
+        self,
+        input_fn: Callable[[str], str] = click.prompt,
+        echo_fn: Callable[[str], str] = click.secho,
+        confirm_fn: Callable[[str], bool] = click.confirm,
+        load_application_fn: Callable[[str], Application] = load_application,
+        get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
+        check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
+        check_image_exists_fn: Callable[[str], str] = check_image_exists,
+        get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
+        list_latest_images_fn: Callable[[str], str] = list_latest_images,
+        start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
+        check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
+        subprocess: Callable[[str], str] = subprocess.run,
+    ):
+        self.input_fn = input_fn
+        self.echo_fn = echo_fn
+        self.confirm_fn = confirm_fn
+        self.load_application_fn = load_application_fn
+        self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
+        self.check_codebase_exists_fn = check_codebase_exists_fn
+        self.check_image_exists_fn = check_image_exists_fn
+        self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
+        self.list_latest_images_fn = list_latest_images_fn
+        self.start_build_extraction_fn = start_build_extraction_fn
+        self.check_if_commit_exists_fn = check_if_commit_exists_fn
+        self.subprocess = subprocess
+
+    def prepare(self):
+        """Sets up an application codebase for use within a DBT platform
+        project."""
+        templates = setup_templates()
+
+        repository = (
+            self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
+            .stdout.split("/")[-1]
+            .strip()
+            .removesuffix(".git")
+        )
+        if repository.endswith("-deploy") or Path("./copilot").exists():
+            raise NotInCodeBaseRepositoryError
+
+        builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
+        builder_configuration_response = requests.get(builder_configuration_url)
+        builder_configuration_content = yaml.safe_load(
+            builder_configuration_response.content.decode("utf-8")
+        )
+        builder_versions = next(
+            (
+                item
+                for item in builder_configuration_content["builders"]
+                if item["name"] == "paketobuildpacks/builder-jammy-base"
+            ),
+            None,
+        )
+        builder_version = max(x["version"] for x in builder_versions["versions"])
+        builder_version = min(builder_version, "0.4.240")
+
+        Path("./.copilot/phases").mkdir(parents=True, exist_ok=True)
+        image_build_run_contents = templates.get_template(f".copilot/image_build_run.sh").render()
+
+        config_contents = templates.get_template(f".copilot/config.yml").render(
+            repository=repository, builder_version=builder_version
+        )
+        self.echo_fn(
+            mkfile(
+                Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
+            )
+        )
+
+        image_build_run_file = Path(".copilot/image_build_run.sh")
+        image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
+
+        self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
+
+        for phase in ["build", "install", "post_build", "pre_build"]:
+            phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
+
+            self.echo_fn(
+                mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
+            )
+
+    def build(self, app: str, codebase: str, commit: str):
+        """Trigger a CodePipeline pipeline based build."""
+        session = self.get_aws_session_or_abort_fn()
+        self.load_application_fn(app, default_session=session)
+
+        self.check_if_commit_exists_fn(commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
+            {
+                "projectName": f"codebuild-{app}-{codebase}",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceVersion": commit,
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def deploy(self, app, env, codebase, commit):
+        """Trigger a CodePipeline pipeline based deployment."""
+        session = self.get_aws_session_or_abort_fn()
+
+        application = self.load_application_fn(app, default_session=session)
+        if not application.environments.get(env):
+            raise ApplicationEnvironmentNotFoundError()
+
+        json.loads(self.check_codebase_exists_fn(session, application, codebase))
+
+        self.check_image_exists_fn(session, application, codebase, commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
+            {
+                "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceTypeOverride": "NO_SOURCE",
+                "environmentVariablesOverride": [
+                    {"name": "COPILOT_ENVIRONMENT", "value": env},
+                    {"name": "IMAGE_TAG", "value": f"commit-{commit}"},
+                ],
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                "Your deployment has been triggered. Check your build progress in the AWS Console: "
+                f"{build_url}",
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def list(self, app: str, with_images: bool):
+        """List available codebases for the application."""
+        session = self.get_aws_session_or_abort_fn()
+        application = self.load_application_fn(app, session)
+        ssm_client = session.client("ssm")
+        ecr_client = session.client("ecr")
+        codebases = self.__get_codebases(application, ssm_client)
+
+        self.echo_fn("The following codebases are available:")
+
+        for codebase in codebases:
+            self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
+            if with_images:
+                self.list_latest_images_fn(
+                    ecr_client,
+                    f"{application.name}/{codebase['name']}",
+                    codebase["repository"],
+                    self.echo_fn,
+                )
+
+        self.echo_fn("")
+
+    def __get_codebases(self, application, ssm_client):
+        parameters = ssm_client.get_parameters_by_path(
+            Path=f"/copilot/applications/{application.name}/codebases",
+            Recursive=True,
+        )["Parameters"]
+
+        codebases = [json.loads(p["Value"]) for p in parameters]
+
+        if not codebases:
+            # TODO Is this really an error? Or just no codebases so we could return an empty list?
+            raise NoCopilotCodebasesFoundError
+        return codebases
+
+    def __start_build_with_confirmation(
+        self,
+        confirm_fn,
+        codebuild_client,
+        get_build_url_from_arn_fn,
+        confirmation_message,
+        build_options,
+    ):
+        if confirm_fn(confirmation_message):
+            build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
+            return get_build_url_from_arn_fn(build_arn)
+        return None