dbt-platform-helper 12.0.2__tar.gz → 12.1.0__tar.gz

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 12.0.2
+Version: 12.1.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/COMMANDS.md

@@ -222,7 +222,7 @@ platform-helper codebase build --app <application> --codebase <codebase>
 - `--app <text>`
   - AWS application name
 - `--codebase <text>`
-  - The codebase name as specified in the pipelines.yml file
+  - The codebase name as specified in the platform-config.yml file
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._
@@ -232,8 +232,6 @@ platform-helper codebase build --app <application> --codebase <codebase>
 
 [↩ Parent](#platform-helper-codebase)
 
-    Trigger a CodePipeline pipeline based deployment.
-
 ## Usage
 
 ```
@@ -248,7 +246,7 @@ platform-helper codebase deploy --app <application> --env <environment> --codeba
 - `--env <text>`
   - AWS Copilot environment
 - `--codebase <text>`
-  - The codebase name as specified in the pipelines.yml file
+  - The codebase name as specified in the platform-config.yml file
 - `--commit <text>`
   - GitHub commit hash
 - `--help <boolean>` _Defaults to False._

dbt_platform_helper-12.1.0/dbt_platform_helper/commands/codebase.py

@@ -0,0 +1,145 @@
+import json
+import os
+
+import click
+
+from dbt_platform_helper.domain.codebase import Codebase
+from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
+from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
+from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
+from dbt_platform_helper.exceptions import ImageNotFoundError
+from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
+from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.git import CommitNotFoundError
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def codebase():
+    """Codebase commands."""
+    check_platform_helper_version_needs_update()
+
+
+@codebase.command()
+def prepare():
+    """Sets up an application codebase for use within a DBT platform project."""
+    try:
+        Codebase().prepare()
+    except NotInCodeBaseRepositoryError:
+        # TODO print error attached to exception
+        click.secho(
+            "You are in the deploy repository; make sure you are in the application codebase repository.",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--with-images",
+    help="List up to the last 10 images tagged for this codebase",
+    default=False,
+    is_flag=True,
+)
+def list(app, with_images):
+    """List available codebases for the application."""
+    try:
+        Codebase().list(app, with_images)
+    except NoCopilotCodebasesFoundError:
+        click.secho(
+            f"""No codebases found for application "{app}""",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationNotFoundError:
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def build(app, codebase, commit):
+    """Trigger a CodePipeline pipeline based build."""
+    try:
+        Codebase().build(app, codebase, commit)
+    except ApplicationNotFoundError:
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+    except CommitNotFoundError:
+        click.secho(
+            f'The commit hash "{commit}" either does not exist or you need to run `git fetch`.',
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationDeploymentNotTriggered:
+        click.secho(
+            f"Your build for {codebase} was not triggered.",
+            fg="red",
+        )
+        raise click.Abort
+
+
+@codebase.command()
+@click.option("--app", help="AWS application name", required=True)
+@click.option("--env", help="AWS Copilot environment", required=True)
+@click.option(
+    "--codebase",
+    help="The codebase name as specified in the platform-config.yml file",
+    required=True,
+)
+@click.option("--commit", help="GitHub commit hash", required=True)
+def deploy(app, env, codebase, commit):
+    try:
+        Codebase().deploy(app, env, codebase, commit)
+    except ApplicationNotFoundError:
+        click.secho(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{app}"; ensure you have set the environment variable "AWS_PROFILE" correctly.""",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationEnvironmentNotFoundError:
+        click.secho(
+            f"""The environment "{env}" either does not exist or has not been deployed.""",
+            fg="red",
+        )
+        raise click.Abort
+    # TODO: don't hide json decode error
+    except (
+        CopilotCodebaseNotFoundError,
+        json.JSONDecodeError,
+    ):
+        click.secho(
+            f"""The codebase "{codebase}" either does not exist or has not been deployed.""",
+            fg="red",
+        )
+        raise click.Abort
+    except ImageNotFoundError:
+        click.secho(
+            f'The commit hash "{commit}" has not been built into an image, try the '
+            "`platform-helper codebase build` command first.",
+            fg="red",
+        )
+        raise click.Abort
+    except ApplicationDeploymentNotTriggered:
+        click.secho(
+            f"Your deployment for {codebase} was not triggered.",
+            fg="red",
+        )
+        raise click.Abort

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/constants.py

@@ -3,3 +3,4 @@ PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"
 ENVIRONMENTS_KEY = "environments"
 DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+PLATFORM_HELPER_CACHE_FILE = ".platform-helper-config-cache.yml"

dbt_platform_helper-12.1.0/dbt_platform_helper/domain/codebase.py

@@ -0,0 +1,222 @@
+import json
+import stat
+import subprocess
+from collections.abc import Callable
+from pathlib import Path
+
+import click
+import requests
+import yaml
+from boto3 import Session
+
+from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
+from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
+from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
+from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
+from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.utils.application import load_application
+from dbt_platform_helper.utils.aws import check_codebase_exists
+from dbt_platform_helper.utils.aws import check_image_exists
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.aws import get_build_url_from_arn
+from dbt_platform_helper.utils.aws import list_latest_images
+from dbt_platform_helper.utils.aws import start_build_extraction
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.git import check_if_commit_exists
+from dbt_platform_helper.utils.template import setup_templates
+
+
+class Codebase:
+    def __init__(
+        self,
+        input_fn: Callable[[str], str] = click.prompt,
+        echo_fn: Callable[[str], str] = click.secho,
+        confirm_fn: Callable[[str], bool] = click.confirm,
+        load_application_fn: Callable[[str], Application] = load_application,
+        get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
+        check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
+        check_image_exists_fn: Callable[[str], str] = check_image_exists,
+        get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
+        list_latest_images_fn: Callable[[str], str] = list_latest_images,
+        start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
+        check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
+        subprocess: Callable[[str], str] = subprocess.run,
+    ):
+        self.input_fn = input_fn
+        self.echo_fn = echo_fn
+        self.confirm_fn = confirm_fn
+        self.load_application_fn = load_application_fn
+        self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
+        self.check_codebase_exists_fn = check_codebase_exists_fn
+        self.check_image_exists_fn = check_image_exists_fn
+        self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
+        self.list_latest_images_fn = list_latest_images_fn
+        self.start_build_extraction_fn = start_build_extraction_fn
+        self.check_if_commit_exists_fn = check_if_commit_exists_fn
+        self.subprocess = subprocess
+
+    def prepare(self):
+        """Sets up an application codebase for use within a DBT platform
+        project."""
+        templates = setup_templates()
+
+        repository = (
+            self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
+            .stdout.split("/")[-1]
+            .strip()
+            .removesuffix(".git")
+        )
+        if repository.endswith("-deploy") or Path("./copilot").exists():
+            raise NotInCodeBaseRepositoryError
+
+        builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
+        builder_configuration_response = requests.get(builder_configuration_url)
+        builder_configuration_content = yaml.safe_load(
+            builder_configuration_response.content.decode("utf-8")
+        )
+        builder_versions = next(
+            (
+                item
+                for item in builder_configuration_content["builders"]
+                if item["name"] == "paketobuildpacks/builder-jammy-base"
+            ),
+            None,
+        )
+        builder_version = max(x["version"] for x in builder_versions["versions"])
+        builder_version = min(builder_version, "0.4.240")
+
+        Path("./.copilot/phases").mkdir(parents=True, exist_ok=True)
+        image_build_run_contents = templates.get_template(f".copilot/image_build_run.sh").render()
+
+        config_contents = templates.get_template(f".copilot/config.yml").render(
+            repository=repository, builder_version=builder_version
+        )
+        self.echo_fn(
+            mkfile(
+                Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
+            )
+        )
+
+        image_build_run_file = Path(".copilot/image_build_run.sh")
+        image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
+
+        self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
+
+        for phase in ["build", "install", "post_build", "pre_build"]:
+            phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
+
+            self.echo_fn(
+                mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
+            )
+
+    def build(self, app: str, codebase: str, commit: str):
+        """Trigger a CodePipeline pipeline based build."""
+        session = self.get_aws_session_or_abort_fn()
+        self.load_application_fn(app, default_session=session)
+
+        self.check_if_commit_exists_fn(commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
+            {
+                "projectName": f"codebuild-{app}-{codebase}",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceVersion": commit,
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def deploy(self, app, env, codebase, commit):
+        """Trigger a CodePipeline pipeline based deployment."""
+        session = self.get_aws_session_or_abort_fn()
+
+        application = self.load_application_fn(app, default_session=session)
+        if not application.environments.get(env):
+            raise ApplicationEnvironmentNotFoundError()
+
+        json.loads(self.check_codebase_exists_fn(session, application, codebase))
+
+        self.check_image_exists_fn(session, application, codebase, commit)
+
+        codebuild_client = session.client("codebuild")
+        build_url = self.__start_build_with_confirmation(
+            self.confirm_fn,
+            codebuild_client,
+            self.get_build_url_from_arn_fn,
+            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
+            {
+                "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
+                "artifactsOverride": {"type": "NO_ARTIFACTS"},
+                "sourceTypeOverride": "NO_SOURCE",
+                "environmentVariablesOverride": [
+                    {"name": "COPILOT_ENVIRONMENT", "value": env},
+                    {"name": "IMAGE_TAG", "value": f"commit-{commit}"},
+                ],
+            },
+        )
+
+        if build_url:
+            return self.echo_fn(
+                "Your deployment has been triggered. Check your build progress in the AWS Console: "
+                f"{build_url}",
+            )
+
+        raise ApplicationDeploymentNotTriggered()
+
+    def list(self, app: str, with_images: bool):
+        """List available codebases for the application."""
+        session = self.get_aws_session_or_abort_fn()
+        application = self.load_application_fn(app, session)
+        ssm_client = session.client("ssm")
+        ecr_client = session.client("ecr")
+        codebases = self.__get_codebases(application, ssm_client)
+
+        self.echo_fn("The following codebases are available:")
+
+        for codebase in codebases:
+            self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
+            if with_images:
+                self.list_latest_images_fn(
+                    ecr_client,
+                    f"{application.name}/{codebase['name']}",
+                    codebase["repository"],
+                    self.echo_fn,
+                )
+
+        self.echo_fn("")
+
+    # TODO return empty list without exception
+    def __get_codebases(self, application, ssm_client):
+        parameters = ssm_client.get_parameters_by_path(
+            Path=f"/copilot/applications/{application.name}/codebases",
+            Recursive=True,
+        )["Parameters"]
+
+        codebases = [json.loads(p["Value"]) for p in parameters]
+
+        if not codebases:
+            raise NoCopilotCodebasesFoundError
+        return codebases
+
+    def __start_build_with_confirmation(
+        self,
+        confirm_fn,
+        codebuild_client,
+        get_build_url_from_arn_fn,
+        confirmation_message,
+        build_options,
+    ):
+        if confirm_fn(confirmation_message):
+            build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
+            return get_build_url_from_arn_fn(build_arn)
+        return None

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/domain/database_copy.py

@@ -8,9 +8,9 @@ from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
 from dbt_platform_helper.exceptions import AWSException
 from dbt_platform_helper.utils.application import Application
-from dbt_platform_helper.utils.application import ApplicationNotFoundError
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import Vpc
 from dbt_platform_helper.utils.aws import get_connection_string

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/exceptions.py

@@ -18,3 +18,31 @@ class IncompatibleMinorVersion(ValidationException):
         super().__init__()
         self.app_version = app_version
         self.check_version = check_version
+
+
+class CopilotCodebaseNotFoundError(Exception):
+    pass
+
+
+class NotInCodeBaseRepositoryError(Exception):
+    pass
+
+
+class NoCopilotCodebasesFoundError(Exception):
+    pass
+
+
+class ImageNotFoundError(Exception):
+    pass
+
+
+class ApplicationDeploymentNotTriggered(Exception):
+    pass
+
+
+class ApplicationNotFoundError(Exception):
+    pass
+
+
+class ApplicationEnvironmentNotFoundError(Exception):
+    pass

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/utils/application.py

@@ -8,6 +8,7 @@ import yaml
 from boto3 import Session
 from yaml.parser import ParserError
 
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
 from dbt_platform_helper.utils.aws import get_ssm_secrets
@@ -67,10 +68,6 @@ class Application:
         return str(self) == str(other)
 
 
-class ApplicationNotFoundError(Exception):
-    pass
-
-
 def load_application(app: str = None, default_session: Session = None) -> Application:
     application = Application(app if app else get_application_name())
     current_session = default_session if default_session else get_aws_session_or_abort()

{dbt_platform_helper-12.0.2 → dbt_platform_helper-12.1.0}/dbt_platform_helper/utils/aws.py

@@ -13,7 +13,12 @@ import yaml
 from boto3 import Session
 
 from dbt_platform_helper.exceptions import AWSException
+from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
+from dbt_platform_helper.exceptions import ImageNotFoundError
 from dbt_platform_helper.exceptions import ValidationException
+from dbt_platform_helper.utils.files import cache_refresh_required
+from dbt_platform_helper.utils.files import read_supported_versions_from_cache
+from dbt_platform_helper.utils.files import write_to_cache
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
 SSM_PATH = "/copilot/{app}/{env}/secrets/{name}"
@@ -351,6 +356,59 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
     return parameter_data
 
 
+def get_supported_redis_versions():
+
+    if cache_refresh_required("redis"):
+
+        supported_versions = []
+
+        session = get_aws_session_or_abort()
+        elasticache_client = session.client("elasticache")
+
+        supported_versions_response = elasticache_client.describe_cache_engine_versions(
+            Engine="redis"
+        )
+
+        supported_versions = [
+            version["EngineVersion"]
+            for version in supported_versions_response["CacheEngineVersions"]
+        ]
+
+        write_to_cache("redis", supported_versions)
+
+        return supported_versions
+
+    else:
+        return read_supported_versions_from_cache("redis")
+
+
+def get_supported_opensearch_versions():
+
+    if cache_refresh_required("opensearch"):
+
+        supported_versions = []
+
+        session = get_aws_session_or_abort()
+        opensearch_client = session.client("opensearch")
+
+        response = opensearch_client.list_versions()
+        all_versions = response["Versions"]
+
+        opensearch_versions = [
+            version for version in all_versions if not version.startswith("Elasticsearch_")
+        ]
+        supported_versions = [
+            version.removeprefix("OpenSearch_") for version in opensearch_versions
+        ]
+
+        write_to_cache("opensearch", supported_versions)
+
+        return supported_versions
+
+    else:
+        return read_supported_versions_from_cache("opensearch")
+
+
 def get_connection_string(
     session: Session,
     app: str,
@@ -420,3 +478,77 @@ def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) ->
         raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
 
     return Vpc(subnets, sec_groups)
+
+
+def start_build_extraction(codebuild_client, build_options):
+    response = codebuild_client.start_build(**build_options)
+    return response["build"]["arn"]
+
+
+def check_codebase_exists(session: Session, application, codebase: str):
+    try:
+        ssm_client = session.client("ssm")
+        ssm_client.get_parameter(
+            Name=f"/copilot/applications/{application.name}/codebases/{codebase}"
+        )["Parameter"]["Value"]
+    except (
+        KeyError,
+        ValueError,
+        ssm_client.exceptions.ParameterNotFound,
+    ):
+        raise CopilotCodebaseNotFoundError
+
+
+def check_image_exists(session, application, codebase, commit):
+    ecr_client = session.client("ecr")
+    try:
+        ecr_client.describe_images(
+            repositoryName=f"{application.name}/{codebase}",
+            imageIds=[{"imageTag": f"commit-{commit}"}],
+        )
+    except (
+        ecr_client.exceptions.RepositoryNotFoundException,
+        ecr_client.exceptions.ImageNotFoundException,
+    ):
+        raise ImageNotFoundError
+
+
+def get_build_url_from_arn(build_arn: str) -> str:
+    _, _, _, region, account_id, project_name, build_id = build_arn.split(":")
+    project_name = project_name.removeprefix("build/")
+    return (
+        f"https://eu-west-2.console.aws.amazon.com/codesuite/codebuild/{account_id}/projects/"
+        f"{project_name}/build/{project_name}%3A{build_id}"
+    )
+
+
+def list_latest_images(ecr_client, ecr_repository_name, codebase_repository, echo_fn):
+    paginator = ecr_client.get_paginator("describe_images")
+    describe_images_response_iterator = paginator.paginate(
+        repositoryName=ecr_repository_name,
+        filter={"tagStatus": "TAGGED"},
+    )
+    images = []
+    for page in describe_images_response_iterator:
+        images += page["imageDetails"]
+
+    sorted_images = sorted(
+        images,
+        key=lambda i: i["imagePushedAt"],
+        reverse=True,
+    )
+
+    MAX_RESULTS = 20
+
+    for image in sorted_images[:MAX_RESULTS]:
+        try:
+            commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
+            if not commit_tag:
+                continue
+
+            commit_hash = commit_tag.replace("commit-", "")
+            echo_fn(
+                f"  - https://github.com/{codebase_repository}/commit/{commit_hash} - published: {image['imagePushedAt']}"
+            )
+        except StopIteration:
+            continue