dbt-platform-helper 12.0.2__py3-none-any.whl → 12.2.0__py3-none-any.whl

dbt_platform_helper/commands/conduit.py

@@ -1,453 +1,78 @@
-import json
-import random
-import string
-import subprocess
-import time
-
 import click
-from botocore.exceptions import ClientError
-from cfn_tools import dump_yaml
-from cfn_tools import load_yaml
 
-from dbt_platform_helper.utils.application import Application
+from dbt_platform_helper.constants import CONDUIT_ADDON_TYPES
+from dbt_platform_helper.domain.conduit import Conduit
+from dbt_platform_helper.exceptions import AddonNotFoundError
+from dbt_platform_helper.exceptions import AddonTypeMissingFromConfigError
+from dbt_platform_helper.exceptions import CreateTaskTimeoutError
+from dbt_platform_helper.exceptions import InvalidAddonTypeError
+from dbt_platform_helper.exceptions import NoClusterError
+from dbt_platform_helper.exceptions import ParameterNotFoundError
+from dbt_platform_helper.providers.secrets import SecretNotFoundError
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import (
-    get_postgres_connection_data_updated_with_master_secret,
-)
 from dbt_platform_helper.utils.click import ClickDocOptCommand
-from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
 
-
-class ConduitError(Exception):
-    pass
-
-
-class InvalidAddonTypeConduitError(ConduitError):
-    def __init__(self, addon_type):
-        self.addon_type = addon_type
-
-
-class NoClusterConduitError(ConduitError):
-    pass
-
-
-class SecretNotFoundConduitError(ConduitError):
-    pass
-
-
-class CreateTaskTimeoutConduitError(ConduitError):
-    pass
-
-
-class ParameterNotFoundConduitError(ConduitError):
-    pass
-
-
-class AddonNotFoundConduitError(ConduitError):
-    pass
-
-
-CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"
-CONDUIT_ADDON_TYPES = [
-    "opensearch",
-    "postgres",
-    "redis",
-]
 CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
 
 
-def normalise_secret_name(addon_name: str) -> str:
-    return addon_name.replace("-", "_").upper()
-
-
-def get_addon_type(app: Application, env: str, addon_name: str) -> str:
-    session = app.environments[env].session
-    ssm_client = session.client("ssm")
-    addon_type = None
-
-    try:
-        addon_config = json.loads(
-            ssm_client.get_parameter(
-                Name=f"/copilot/applications/{app.name}/environments/{env}/addons"
-            )["Parameter"]["Value"]
-        )
-    except ssm_client.exceptions.ParameterNotFound:
-        raise ParameterNotFoundConduitError
-
-    if addon_name not in addon_config.keys():
-        raise AddonNotFoundConduitError
-
-    for name, config in addon_config.items():
-        if name == addon_name:
-            addon_type = config["type"]
-
-    if not addon_type or addon_type not in CONDUIT_ADDON_TYPES:
-        raise InvalidAddonTypeConduitError(addon_type)
-
-    if "postgres" in addon_type:
-        addon_type = "postgres"
-
-    return addon_type
-
-
-def get_parameter_name(
-    app: Application, env: str, addon_type: str, addon_name: str, access: str
-) -> str:
-    if addon_type == "postgres":
-        return f"/copilot/{app.name}/{env}/conduits/{normalise_secret_name(addon_name)}_{access.upper()}"
-    elif addon_type == "redis" or addon_type == "opensearch":
-        return f"/copilot/{app.name}/{env}/conduits/{normalise_secret_name(addon_name)}_ENDPOINT"
-    else:
-        return f"/copilot/{app.name}/{env}/conduits/{normalise_secret_name(addon_name)}"
-
-
-def get_or_create_task_name(
-    app: Application, env: str, addon_name: str, parameter_name: str
-) -> str:
-    ssm = app.environments[env].session.client("ssm")
-
-    try:
-        return ssm.get_parameter(Name=parameter_name)["Parameter"]["Value"]
-    except ssm.exceptions.ParameterNotFound:
-        random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
-        return f"conduit-{app.name}-{env}-{addon_name}-{random_id}"
-
-
-def get_cluster_arn(app: Application, env: str) -> str:
-    ecs_client = app.environments[env].session.client("ecs")
-
-    for cluster_arn in ecs_client.list_clusters()["clusterArns"]:
-        tags_response = ecs_client.list_tags_for_resource(resourceArn=cluster_arn)
-        tags = tags_response["tags"]
-
-        app_key_found = False
-        env_key_found = False
-        cluster_key_found = False
-
-        for tag in tags:
-            if tag["key"] == "copilot-application" and tag["value"] == app.name:
-                app_key_found = True
-            if tag["key"] == "copilot-environment" and tag["value"] == env:
-                env_key_found = True
-            if tag["key"] == "aws:cloudformation:logical-id" and tag["value"] == "Cluster":
-                cluster_key_found = True
-
-        if app_key_found and env_key_found and cluster_key_found:
-            return cluster_arn
-
-    raise NoClusterConduitError
-
-
-def get_connection_secret_arn(app: Application, env: str, secret_name: str) -> str:
-    secrets_manager = app.environments[env].session.client("secretsmanager")
-    ssm = app.environments[env].session.client("ssm")
-
-    try:
-        return ssm.get_parameter(Name=secret_name, WithDecryption=False)["Parameter"]["ARN"]
-    except ssm.exceptions.ParameterNotFound:
-        pass
-
-    try:
-        return secrets_manager.describe_secret(SecretId=secret_name)["ARN"]
-    except secrets_manager.exceptions.ResourceNotFoundException:
-        pass
-
-    raise SecretNotFoundConduitError(secret_name)
-
-
-def create_postgres_admin_task(
-    app: Application, env: str, secret_name: str, task_name: str, addon_type: str, addon_name: str
-):
-    session = app.environments[env].session
-    read_only_secret_name = secret_name + "_READ_ONLY_USER"
-    master_secret_name = (
-        f"/copilot/{app.name}/{env}/secrets/{normalise_secret_name(addon_name)}_RDS_MASTER_ARN"
-    )
-    master_secret_arn = session.client("ssm").get_parameter(
-        Name=master_secret_name, WithDecryption=True
-    )["Parameter"]["Value"]
-    connection_string = json.dumps(
-        get_postgres_connection_data_updated_with_master_secret(
-            session, read_only_secret_name, master_secret_arn
-        )
-    )
-
-    subprocess.call(
-        f"copilot task run --app {app.name} --env {env} "
-        f"--task-group-name {task_name} "
-        f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
-        f"--env-vars CONNECTION_SECRET='{connection_string}' "
-        "--platform-os linux "
-        "--platform-arch arm64",
-        shell=True,
-    )
-
-
-def create_addon_client_task(
-    app: Application,
-    env: str,
-    addon_type: str,
-    addon_name: str,
-    task_name: str,
-    access: str,
-):
-    secret_name = f"/copilot/{app.name}/{env}/secrets/{normalise_secret_name(addon_name)}"
-    session = app.environments[env].session
-
-    if addon_type == "postgres":
-        if access == "read":
-            secret_name += "_READ_ONLY_USER"
-        elif access == "write":
-            secret_name += "_APPLICATION_USER"
-        elif access == "admin":
-            create_postgres_admin_task(app, env, secret_name, task_name, addon_type, addon_name)
-            return
-    elif addon_type == "redis" or addon_type == "opensearch":
-        secret_name += "_ENDPOINT"
-
-    role_name = f"{addon_name}-{app.name}-{env}-conduitEcsTask"
-
-    try:
-        session.client("iam").get_role(RoleName=role_name)
-        execution_role = f"--execution-role {role_name} "
-    except ClientError as ex:
-        execution_role = ""
-        # We cannot check for botocore.errorfactory.NoSuchEntityException as botocore generates that class on the fly as part of errorfactory.
-        # factory. Checking the error code is the recommended way of handling these exceptions.
-        if ex.response.get("Error", {}).get("Code", None) != "NoSuchEntity":
-            abort_with_error(
-                f"cannot obtain Role {role_name}: {ex.response.get('Error', {}).get('Message', '')}"
-            )
-
-    subprocess.call(
-        f"copilot task run --app {app.name} --env {env} "
-        f"--task-group-name {task_name} "
-        f"{execution_role}"
-        f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
-        f"--secrets CONNECTION_SECRET={get_connection_secret_arn(app, env, secret_name)} "
-        "--platform-os linux "
-        "--platform-arch arm64",
-        shell=True,
-    )
-
-
-def addon_client_is_running(app: Application, env: str, cluster_arn: str, task_name: str) -> bool:
-    ecs_client = app.environments[env].session.client("ecs")
-
-    tasks = ecs_client.list_tasks(
-        cluster=cluster_arn,
-        desiredStatus="RUNNING",
-        family=f"copilot-{task_name}",
-    )
-
-    if not tasks["taskArns"]:
-        return False
-
-    described_tasks = ecs_client.describe_tasks(cluster=cluster_arn, tasks=tasks["taskArns"])
-
-    # The ExecuteCommandAgent often takes longer to start running than the task and without the
-    # agent it's not possible to exec into a task.
-    for task in described_tasks["tasks"]:
-        for container in task["containers"]:
-            for agent in container["managedAgents"]:
-                if agent["name"] == "ExecuteCommandAgent" and agent["lastStatus"] == "RUNNING":
-                    return True
-
-    return False
-
-
-def connect_to_addon_client_task(app: Application, env: str, cluster_arn: str, task_name: str):
-    tries = 0
-    running = False
-
-    while tries < 15 and not running:
-        tries += 1
-
-        if addon_client_is_running(app, env, cluster_arn, task_name):
-            running = True
-            subprocess.call(
-                "copilot task exec "
-                f"--app {app.name} --env {env} "
-                f"--name {task_name} "
-                f"--command bash",
-                shell=True,
-            )
-
-        time.sleep(1)
-
-    if not running:
-        raise CreateTaskTimeoutConduitError
-
-
-def add_stack_delete_policy_to_task_role(app: Application, env: str, task_name: str):
-    session = app.environments[env].session
-    cloudformation_client = session.client("cloudformation")
-    iam_client = session.client("iam")
-
-    conduit_stack_name = f"task-{task_name}"
-    conduit_stack_resources = cloudformation_client.list_stack_resources(
-        StackName=conduit_stack_name
-    )["StackResourceSummaries"]
-
-    for resource in conduit_stack_resources:
-        if resource["LogicalResourceId"] == "DefaultTaskRole":
-            task_role_name = resource["PhysicalResourceId"]
-            iam_client.put_role_policy(
-                RoleName=task_role_name,
-                PolicyName="DeleteCloudFormationStack",
-                PolicyDocument=json.dumps(
-                    {
-                        "Version": "2012-10-17",
-                        "Statement": [
-                            {
-                                "Action": ["cloudformation:DeleteStack"],
-                                "Effect": "Allow",
-                                "Resource": f"arn:aws:cloudformation:*:*:stack/{conduit_stack_name}/*",
-                            },
-                        ],
-                    },
-                ),
-            )
-
-
-def update_conduit_stack_resources(
-    app: Application,
-    env: str,
-    addon_type: str,
-    addon_name: str,
-    task_name: str,
-    parameter_name: str,
-    access: str,
-):
-    session = app.environments[env].session
-    cloudformation_client = session.client("cloudformation")
-
-    conduit_stack_name = f"task-{task_name}"
-    template = cloudformation_client.get_template(StackName=conduit_stack_name)
-    template_yml = load_yaml(template["TemplateBody"])
-    template_yml["Resources"]["LogGroup"]["DeletionPolicy"] = "Retain"
-    template_yml["Resources"]["TaskNameParameter"] = load_yaml(
-        f"""
-        Type: AWS::SSM::Parameter
-        Properties:
-          Name: {parameter_name}
-          Type: String
-          Value: {task_name}
-        """
-    )
-
-    iam_client = session.client("iam")
-    log_filter_role_arn = iam_client.get_role(RoleName="CWLtoSubscriptionFilterRole")["Role"]["Arn"]
-
-    ssm_client = session.client("ssm")
-    destination_log_group_arns = json.loads(
-        ssm_client.get_parameter(Name="/copilot/tools/central_log_groups")["Parameter"]["Value"]
-    )
-
-    destination_arn = destination_log_group_arns["dev"]
-    if env.lower() in ("prod", "production"):
-        destination_arn = destination_log_group_arns["prod"]
-
-    template_yml["Resources"]["SubscriptionFilter"] = load_yaml(
-        f"""
-        Type: AWS::Logs::SubscriptionFilter
-        DeletionPolicy: Retain
-        Properties:
-          RoleArn: {log_filter_role_arn}
-          LogGroupName: /copilot/{task_name}
-          FilterName: /copilot/conduit/{app.name}/{env}/{addon_type}/{addon_name}/{task_name.rsplit("-", 1)[1]}/{access}
-          FilterPattern: ''
-          DestinationArn: {destination_arn}
-        """
-    )
-
-    params = []
-    if "Parameters" in template_yml:
-        for param in template_yml["Parameters"]:
-            params.append({"ParameterKey": param, "UsePreviousValue": True})
-
-    cloudformation_client.update_stack(
-        StackName=conduit_stack_name,
-        TemplateBody=dump_yaml(template_yml),
-        Parameters=params,
-        Capabilities=["CAPABILITY_IAM"],
-    )
-
-
-def start_conduit(
-    application: Application,
-    env: str,
-    addon_type: str,
-    addon_name: str,
-    access: str = "read",
-):
-    cluster_arn = get_cluster_arn(application, env)
-    parameter_name = get_parameter_name(application, env, addon_type, addon_name, access)
-    task_name = get_or_create_task_name(application, env, addon_name, parameter_name)
-
-    if not addon_client_is_running(application, env, cluster_arn, task_name):
-        create_addon_client_task(application, env, addon_type, addon_name, task_name, access)
-        add_stack_delete_policy_to_task_role(application, env, task_name)
-        update_conduit_stack_resources(
-            application, env, addon_type, addon_name, task_name, parameter_name, access
-        )
-
-    connect_to_addon_client_task(application, env, cluster_arn, task_name)
-
-
 @click.command(cls=ClickDocOptCommand)
 @click.argument("addon_name", type=str, required=True)
-@click.option("--app", help="AWS application name", required=True)
-@click.option("--env", help="AWS environment name", required=True)
+@click.option("--app", help="Application name", required=True)
+@click.option("--env", help="Environment name", required=True)
 @click.option(
     "--access",
     default="read",
     type=click.Choice(CONDUIT_ACCESS_OPTIONS),
-    help="Allow write or admin access to database addons",
+    help="Allow read, write or admin access to the database addons.",
 )
 def conduit(addon_name: str, app: str, env: str, access: str):
-    """Create a conduit connection to an addon."""
+    """Opens a shell for a given addon_name create a conduit connection to
+    interact with postgres, opensearch or redis."""
     check_platform_helper_version_needs_update()
     application = load_application(app)
 
     try:
-        addon_type = get_addon_type(application, env, addon_name)
-    except ParameterNotFoundConduitError:
+        Conduit(application).start(env, addon_name, access)
+    except NoClusterError:
+        # TODO: Set exception message in the exceptions and just output the message in the command code, should be able to catch all errors in one block
+        click.secho(f"""No ECS cluster found for "{app}" in "{env}" environment.""", fg="red")
+        exit(1)
+    except SecretNotFoundError as err:
         click.secho(
-            f"""No parameter called "/copilot/applications/{app}/environments/{env}/addons". Try deploying the "{app}" "{env}" environment.""",
+            f"""No secret called "{err}" for "{app}" in "{env}" environment.""",
             fg="red",
         )
         exit(1)
-    except AddonNotFoundConduitError:
+    except CreateTaskTimeoutError:
         click.secho(
-            f"""Addon "{addon_name}" does not exist.""",
+            f"""Client ({addon_name}) ECS task has failed to start for "{app}" in "{env}" environment.""",
             fg="red",
         )
         exit(1)
-    except InvalidAddonTypeConduitError as err:
+    except ParameterNotFoundError:
         click.secho(
-            f"""Addon type "{err.addon_type}" is not supported, we support: {", ".join(CONDUIT_ADDON_TYPES)}.""",
+            f"""No parameter called "/copilot/applications/{app}/environments/{env}/addons". Try deploying the "{app}" "{env}" environment.""",
             fg="red",
         )
         exit(1)
-
-    try:
-        start_conduit(application, env, addon_type, addon_name, access)
-    except NoClusterConduitError:
-        click.secho(f"""No ECS cluster found for "{app}" in "{env}" environment.""", fg="red")
+    except AddonNotFoundError:
+        click.secho(
+            f"""Addon "{addon_name}" does not exist.""",
+            fg="red",
+        )
         exit(1)
-    except SecretNotFoundConduitError as err:
+    except InvalidAddonTypeError as err:
         click.secho(
-            f"""No secret called "{err}" for "{app}" in "{env}" environment.""",
+            f"""Addon type "{err.addon_type}" is not supported, we support: {", ".join(CONDUIT_ADDON_TYPES)}.""",
             fg="red",
         )
         exit(1)
-    except CreateTaskTimeoutConduitError:
+    except AddonTypeMissingFromConfigError:
         click.secho(
-            f"""Client ({addon_name}) ECS task has failed to start for "{app}" in "{env}" environment.""",
+            f"""The configuration for the addon {addon_name}, is missconfigured and missing the addon type.""",
             fg="red",
         )
         exit(1)

dbt_platform_helper/commands/secrets.py

@@ -102,7 +102,7 @@ def list(app, env):
     params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
     secrets = []
 
-    # TODO: refactor shared code with get_ssm_secret_names
+    # TODO: refactor shared code with get_ssm_secret_names - Check if this is still valid
     while True:
         response = client.get_parameters_by_path(**params)
 

dbt_platform_helper/constants.py

@@ -1,5 +1,16 @@
 PLATFORM_CONFIG_FILE = "platform-config.yml"
 PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
+DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+PLATFORM_HELPER_CACHE_FILE = ".platform-helper-config-cache.yml"
+
+# Keys
 CODEBASE_PIPELINES_KEY = "codebase_pipelines"
 ENVIRONMENTS_KEY = "environments"
-DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
+
+# Conduit
+CONDUIT_ADDON_TYPES = [
+    "opensearch",
+    "postgres",
+    "redis",
+]
+CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"