dbt-platform-helper 12.0.2__py3-none-any.whl → 12.2.0__py3-none-any.whl

dbt_platform_helper/providers/copilot.py

@@ -0,0 +1,144 @@
+import json
+import time
+
+from botocore.exceptions import ClientError
+
+from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
+from dbt_platform_helper.exceptions import CreateTaskTimeoutError
+from dbt_platform_helper.providers.ecs import get_ecs_task_arns
+from dbt_platform_helper.providers.secrets import get_connection_secret_arn
+from dbt_platform_helper.providers.secrets import (
+    get_postgres_connection_data_updated_with_master_secret,
+)
+from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.utils.messages import abort_with_error
+
+
+def create_addon_client_task(
+    iam_client,
+    ssm_client,
+    secrets_manager_client,
+    subprocess,
+    application: Application,
+    env: str,
+    addon_type: str,
+    addon_name: str,
+    task_name: str,
+    access: str,
+):
+    secret_name = f"/copilot/{application.name}/{env}/secrets/{_normalise_secret_name(addon_name)}"
+
+    if addon_type == "postgres":
+        if access == "read":
+            secret_name += "_READ_ONLY_USER"
+        elif access == "write":
+            secret_name += "_APPLICATION_USER"
+        elif access == "admin":
+            create_postgres_admin_task(
+                ssm_client,
+                secrets_manager_client,
+                subprocess,
+                application,
+                addon_name,
+                addon_type,
+                env,
+                secret_name,
+                task_name,
+            )
+            return
+    elif addon_type == "redis" or addon_type == "opensearch":
+        secret_name += "_ENDPOINT"
+
+    role_name = f"{addon_name}-{application.name}-{env}-conduitEcsTask"
+
+    try:
+        iam_client.get_role(RoleName=role_name)
+        execution_role = f"--execution-role {role_name} "
+    except ClientError as ex:
+        execution_role = ""
+        # We cannot check for botocore.errorfactory.NoSuchEntityException as botocore generates that class on the fly as part of errorfactory.
+        # factory. Checking the error code is the recommended way of handling these exceptions.
+        if ex.response.get("Error", {}).get("Code", None) != "NoSuchEntity":
+            # TODO Raise an exception to be caught at the command layer
+            abort_with_error(
+                f"cannot obtain Role {role_name}: {ex.response.get('Error', {}).get('Message', '')}"
+            )
+
+    subprocess.call(
+        f"copilot task run --app {application.name} --env {env} "
+        f"--task-group-name {task_name} "
+        f"{execution_role}"
+        f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
+        f"--secrets CONNECTION_SECRET={get_connection_secret_arn(ssm_client,secrets_manager_client, secret_name)} "
+        "--platform-os linux "
+        "--platform-arch arm64",
+        shell=True,
+    )
+
+
+def create_postgres_admin_task(
+    ssm_client,
+    secrets_manager_client,
+    subprocess,
+    app: Application,
+    addon_name: str,
+    addon_type: str,
+    env: str,
+    secret_name: str,
+    task_name: str,
+):
+    read_only_secret_name = secret_name + "_READ_ONLY_USER"
+    master_secret_name = (
+        f"/copilot/{app.name}/{env}/secrets/{_normalise_secret_name(addon_name)}_RDS_MASTER_ARN"
+    )
+    master_secret_arn = ssm_client.get_parameter(Name=master_secret_name, WithDecryption=True)[
+        "Parameter"
+    ]["Value"]
+    connection_string = json.dumps(
+        get_postgres_connection_data_updated_with_master_secret(
+            ssm_client, secrets_manager_client, read_only_secret_name, master_secret_arn
+        )
+    )
+
+    subprocess.call(
+        f"copilot task run --app {app.name} --env {env} "
+        f"--task-group-name {task_name} "
+        f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
+        f"--env-vars CONNECTION_SECRET='{connection_string}' "
+        "--platform-os linux "
+        "--platform-arch arm64",
+        shell=True,
+    )
+
+
+def connect_to_addon_client_task(
+    ecs_client,
+    subprocess,
+    application_name,
+    env,
+    cluster_arn,
+    task_name,
+    addon_client_is_running_fn=get_ecs_task_arns,
+):
+    running = False
+    tries = 0
+    while tries < 15 and not running:
+        tries += 1
+        if addon_client_is_running_fn(ecs_client, cluster_arn, task_name):
+            subprocess.call(
+                "copilot task exec "
+                f"--app {application_name} --env {env} "
+                f"--name {task_name} "
+                f"--command bash",
+                shell=True,
+            )
+            running = True
+
+        time.sleep(1)
+
+    if not running:
+        raise CreateTaskTimeoutError
+
+
+def _normalise_secret_name(addon_name: str) -> str:
+    return addon_name.replace("-", "_").upper()

dbt_platform_helper/providers/ecs.py

@@ -0,0 +1,78 @@
+import random
+import string
+import time
+from typing import List
+
+from dbt_platform_helper.exceptions import ECSAgentNotRunning
+from dbt_platform_helper.exceptions import NoClusterError
+
+
+def get_cluster_arn(ecs_client, application_name: str, env: str) -> str:
+    for cluster_arn in ecs_client.list_clusters()["clusterArns"]:
+        tags_response = ecs_client.list_tags_for_resource(resourceArn=cluster_arn)
+        tags = tags_response["tags"]
+
+        app_key_found = False
+        env_key_found = False
+        cluster_key_found = False
+
+        for tag in tags:
+            if tag["key"] == "copilot-application" and tag["value"] == application_name:
+                app_key_found = True
+            if tag["key"] == "copilot-environment" and tag["value"] == env:
+                env_key_found = True
+            if tag["key"] == "aws:cloudformation:logical-id" and tag["value"] == "Cluster":
+                cluster_key_found = True
+
+        if app_key_found and env_key_found and cluster_key_found:
+            return cluster_arn
+
+    raise NoClusterError
+
+
+def get_or_create_task_name(
+    ssm_client, application_name: str, env: str, addon_name: str, parameter_name: str
+) -> str:
+    try:
+        return ssm_client.get_parameter(Name=parameter_name)["Parameter"]["Value"]
+    except ssm_client.exceptions.ParameterNotFound:
+        random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
+        return f"conduit-{application_name}-{env}-{addon_name}-{random_id}"
+
+
+def get_ecs_task_arns(ecs_client, cluster_arn: str, task_name: str):
+
+    tasks = ecs_client.list_tasks(
+        cluster=cluster_arn,
+        desiredStatus="RUNNING",
+        family=f"copilot-{task_name}",
+    )
+
+    if not tasks["taskArns"]:
+        return []
+
+    return tasks["taskArns"]
+
+
+def ecs_exec_is_available(ecs_client, cluster_arn: str, task_arns: List[str]):
+
+    current_attemps = 0
+    execute_command_agent_status = ""
+
+    while execute_command_agent_status != "RUNNING" and current_attemps < 25:
+
+        current_attemps += 1
+
+        task_details = ecs_client.describe_tasks(cluster=cluster_arn, tasks=task_arns)
+
+        managed_agents = task_details["tasks"][0]["containers"][0]["managedAgents"]
+        execute_command_agent_status = [
+            agent["lastStatus"]
+            for agent in managed_agents
+            if agent["name"] == "ExecuteCommandAgent"
+        ][0]
+
+        time.sleep(1)
+
+    if execute_command_agent_status != "RUNNING":
+        raise ECSAgentNotRunning

dbt_platform_helper/providers/secrets.py

@@ -0,0 +1,85 @@
+import json
+import urllib
+
+from dbt_platform_helper.constants import CONDUIT_ADDON_TYPES
+from dbt_platform_helper.exceptions import AddonNotFoundError
+from dbt_platform_helper.exceptions import AddonTypeMissingFromConfigError
+from dbt_platform_helper.exceptions import InvalidAddonTypeError
+from dbt_platform_helper.exceptions import ParameterNotFoundError
+from dbt_platform_helper.exceptions import SecretNotFoundError
+
+
+def get_postgres_connection_data_updated_with_master_secret(
+    ssm_client, secrets_manager_client, parameter_name, secret_arn
+):
+    response = ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)
+    parameter_value = response["Parameter"]["Value"]
+
+    parameter_data = json.loads(parameter_value)
+
+    secret_response = secrets_manager_client.get_secret_value(SecretId=secret_arn)
+    secret_value = json.loads(secret_response["SecretString"])
+
+    parameter_data["username"] = urllib.parse.quote(secret_value["username"])
+    parameter_data["password"] = urllib.parse.quote(secret_value["password"])
+
+    return parameter_data
+
+
+def get_connection_secret_arn(ssm_client, secrets_manager_client, secret_name: str) -> str:
+
+    try:
+        return ssm_client.get_parameter(Name=secret_name, WithDecryption=False)["Parameter"]["ARN"]
+    except ssm_client.exceptions.ParameterNotFound:
+        pass
+
+    try:
+        return secrets_manager_client.describe_secret(SecretId=secret_name)["ARN"]
+    except secrets_manager_client.exceptions.ResourceNotFoundException:
+        pass
+
+    raise SecretNotFoundError(secret_name)
+
+
+def get_addon_type(ssm_client, application_name: str, env: str, addon_name: str) -> str:
+    addon_type = None
+    try:
+        addon_config = json.loads(
+            ssm_client.get_parameter(
+                Name=f"/copilot/applications/{application_name}/environments/{env}/addons"
+            )["Parameter"]["Value"]
+        )
+    except ssm_client.exceptions.ParameterNotFound:
+        raise ParameterNotFoundError
+
+    if addon_name not in addon_config.keys():
+        raise AddonNotFoundError
+
+    for name, config in addon_config.items():
+        if name == addon_name:
+            if not config.get("type"):
+                raise AddonTypeMissingFromConfigError()
+            addon_type = config["type"]
+
+    if not addon_type or addon_type not in CONDUIT_ADDON_TYPES:
+        raise InvalidAddonTypeError(addon_type)
+
+    if "postgres" in addon_type:
+        addon_type = "postgres"
+
+    return addon_type
+
+
+def get_parameter_name(
+    application_name: str, env: str, addon_type: str, addon_name: str, access: str
+) -> str:
+    if addon_type == "postgres":
+        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}_{access.upper()}"
+    elif addon_type == "redis" or addon_type == "opensearch":
+        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}_ENDPOINT"
+    else:
+        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}"
+
+
+def _normalise_secret_name(addon_name: str) -> str:
+    return addon_name.replace("-", "_").upper()

dbt_platform_helper/templates/addons/svc/prometheus-policy.yml

@@ -1,3 +1,5 @@
+# {% extra_header %}
+# {% version_info %}
 Parameters:
   App:
     Type: String

dbt_platform_helper/templates/pipelines/environments/manifest.yml

@@ -16,7 +16,6 @@ source:
   provider: GitHub
   # Additional properties that further specify the location of the artifacts.
   properties:
-    # Todo: Allow for overriding this, but without risking deploying a branch to higher environments
     branch: main
     repository: https://github.com/{{ git_repo }}
     connection_name: {{ app_name }}

dbt_platform_helper/utils/application.py

@@ -8,6 +8,7 @@ import yaml
 from boto3 import Session
 from yaml.parser import ParserError
 
+from dbt_platform_helper.exceptions import ApplicationNotFoundError
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
 from dbt_platform_helper.utils.aws import get_ssm_secrets
@@ -67,10 +68,6 @@ class Application:
         return str(self) == str(other)
 
 
-class ApplicationNotFoundError(Exception):
-    pass
-
-
 def load_application(app: str = None, default_session: Session = None) -> Application:
     application = Application(app if app else get_application_name())
     current_session = default_session if default_session else get_aws_session_or_abort()

dbt_platform_helper/utils/aws.py

@@ -13,7 +13,12 @@ import yaml
 from boto3 import Session
 
 from dbt_platform_helper.exceptions import AWSException
+from dbt_platform_helper.exceptions import CopilotCodebaseNotFoundError
+from dbt_platform_helper.exceptions import ImageNotFoundError
 from dbt_platform_helper.exceptions import ValidationException
+from dbt_platform_helper.utils.files import cache_refresh_required
+from dbt_platform_helper.utils.files import read_supported_versions_from_cache
+from dbt_platform_helper.utils.files import write_to_cache
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
 SSM_PATH = "/copilot/{app}/{env}/secrets/{name}"
@@ -351,6 +356,59 @@ def get_postgres_connection_data_updated_with_master_secret(session, parameter_n
     return parameter_data
 
 
+def get_supported_redis_versions():
+
+    if cache_refresh_required("redis"):
+
+        supported_versions = []
+
+        session = get_aws_session_or_abort()
+        elasticache_client = session.client("elasticache")
+
+        supported_versions_response = elasticache_client.describe_cache_engine_versions(
+            Engine="redis"
+        )
+
+        supported_versions = [
+            version["EngineVersion"]
+            for version in supported_versions_response["CacheEngineVersions"]
+        ]
+
+        write_to_cache("redis", supported_versions)
+
+        return supported_versions
+
+    else:
+        return read_supported_versions_from_cache("redis")
+
+
+def get_supported_opensearch_versions():
+
+    if cache_refresh_required("opensearch"):
+
+        supported_versions = []
+
+        session = get_aws_session_or_abort()
+        opensearch_client = session.client("opensearch")
+
+        response = opensearch_client.list_versions()
+        all_versions = response["Versions"]
+
+        opensearch_versions = [
+            version for version in all_versions if not version.startswith("Elasticsearch_")
+        ]
+        supported_versions = [
+            version.removeprefix("OpenSearch_") for version in opensearch_versions
+        ]
+
+        write_to_cache("opensearch", supported_versions)
+
+        return supported_versions
+
+    else:
+        return read_supported_versions_from_cache("opensearch")
+
+
 def get_connection_string(
     session: Session,
     app: str,
@@ -420,3 +478,77 @@ def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) ->
         raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
 
     return Vpc(subnets, sec_groups)
+
+
+def start_build_extraction(codebuild_client, build_options):
+    response = codebuild_client.start_build(**build_options)
+    return response["build"]["arn"]
+
+
+def check_codebase_exists(session: Session, application, codebase: str):
+    try:
+        ssm_client = session.client("ssm")
+        ssm_client.get_parameter(
+            Name=f"/copilot/applications/{application.name}/codebases/{codebase}"
+        )["Parameter"]["Value"]
+    except (
+        KeyError,
+        ValueError,
+        ssm_client.exceptions.ParameterNotFound,
+    ):
+        raise CopilotCodebaseNotFoundError
+
+
+def check_image_exists(session, application, codebase, commit):
+    ecr_client = session.client("ecr")
+    try:
+        ecr_client.describe_images(
+            repositoryName=f"{application.name}/{codebase}",
+            imageIds=[{"imageTag": f"commit-{commit}"}],
+        )
+    except (
+        ecr_client.exceptions.RepositoryNotFoundException,
+        ecr_client.exceptions.ImageNotFoundException,
+    ):
+        raise ImageNotFoundError
+
+
+def get_build_url_from_arn(build_arn: str) -> str:
+    _, _, _, region, account_id, project_name, build_id = build_arn.split(":")
+    project_name = project_name.removeprefix("build/")
+    return (
+        f"https://eu-west-2.console.aws.amazon.com/codesuite/codebuild/{account_id}/projects/"
+        f"{project_name}/build/{project_name}%3A{build_id}"
+    )
+
+
+def list_latest_images(ecr_client, ecr_repository_name, codebase_repository, echo_fn):
+    paginator = ecr_client.get_paginator("describe_images")
+    describe_images_response_iterator = paginator.paginate(
+        repositoryName=ecr_repository_name,
+        filter={"tagStatus": "TAGGED"},
+    )
+    images = []
+    for page in describe_images_response_iterator:
+        images += page["imageDetails"]
+
+    sorted_images = sorted(
+        images,
+        key=lambda i: i["imagePushedAt"],
+        reverse=True,
+    )
+
+    MAX_RESULTS = 20
+
+    for image in sorted_images[:MAX_RESULTS]:
+        try:
+            commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
+            if not commit_tag:
+                continue
+
+            commit_hash = commit_tag.replace("commit-", "")
+            echo_fn(
+                f"  - https://github.com/{codebase_repository}/commit/{commit_hash} - published: {image['imagePushedAt']}"
+            )
+        except StopIteration:
+            continue

dbt_platform_helper/utils/files.py

@@ -1,4 +1,6 @@
+import os
 from copy import deepcopy
+from datetime import datetime
 from os import makedirs
 from pathlib import Path
 
@@ -7,6 +9,8 @@ import yaml
 from jinja2 import Environment
 from jinja2 import FileSystemLoader
 
+from dbt_platform_helper.constants import PLATFORM_HELPER_CACHE_FILE
+
 
 def to_yaml(value):
     return yaml.dump(value, sort_keys=False)
@@ -102,3 +106,69 @@ def apply_environment_defaults(config):
     enriched_config["environments"] = defaulted_envs
 
     return enriched_config
+
+
+def read_supported_versions_from_cache(resource_name):
+
+    platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
+
+    return platform_helper_config.get(resource_name).get("versions")
+
+
+def write_to_cache(resource_name, supported_versions):
+
+    platform_helper_config = {}
+
+    if os.path.exists(PLATFORM_HELPER_CACHE_FILE):
+        platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
+
+    cache_dict = {
+        resource_name: {
+            "versions": supported_versions,
+            "date-retrieved": datetime.now().strftime("%d-%m-%y %H:%M:%S"),
+        }
+    }
+
+    platform_helper_config.update(cache_dict)
+
+    with open(PLATFORM_HELPER_CACHE_FILE, "w") as file:
+        file.write("# [!] This file is autogenerated via the platform-helper. Do not edit.\n")
+        yaml.dump(platform_helper_config, file)
+
+
+def cache_refresh_required(resource_name) -> bool:
+    """
+    Checks if the platform-helper should reach out to AWS to 'refresh' its
+    cached values.
+
+    An API call is needed if any of the following conditions are met:
+        1. No cache file (.platform-helper-config.yml) exists.
+        2. The resource name (e.g. redis, opensearch) does not exist within the cache file.
+        3. The date-retrieved value of the cached data is > than a time interval. In this case 1 day.
+    """
+
+    if not os.path.exists(PLATFORM_HELPER_CACHE_FILE):
+        return True
+
+    platform_helper_config = read_file_as_yaml(PLATFORM_HELPER_CACHE_FILE)
+
+    if platform_helper_config.get(resource_name):
+        return check_if_cached_datetime_is_greater_than_interval(
+            platform_helper_config[resource_name].get("date-retrieved"), 1
+        )
+
+    return True
+
+
+def check_if_cached_datetime_is_greater_than_interval(date_retrieved, interval_in_days):
+
+    current_datetime = datetime.now()
+    cached_datetime = datetime.strptime(date_retrieved, "%d-%m-%y %H:%M:%S")
+    delta = current_datetime - cached_datetime
+
+    return delta.days > interval_in_days
+
+
+def read_file_as_yaml(file_name):
+
+    return yaml.safe_load(Path(file_name).read_text())

dbt_platform_helper/utils/git.py

@@ -2,6 +2,10 @@ import re
 import subprocess
 
 
+class CommitNotFoundError(Exception):
+    pass
+
+
 def git_remote():
     git_repo = subprocess.run(
         ["git", "remote", "get-url", "origin"], capture_output=True, text=True
@@ -14,3 +18,12 @@ def extract_repository_name(repository_url):
         return
 
     return re.search(r"([^/:]*/[^/]*)\.git", repository_url).group(1)
+
+
+def check_if_commit_exists(commit):
+    branches_containing_commit = subprocess.run(
+        ["git", "branch", "-r", "--contains", f"{commit}"], capture_output=True, text=True
+    )
+
+    if branches_containing_commit.stderr:
+        raise CommitNotFoundError()