dbt-platform-helper 12.2.4__py3-none-any.whl → 12.4.0__py3-none-any.whl

dbt_platform_helper/providers/copilot.py

@@ -4,12 +4,8 @@ import time
 from botocore.exceptions import ClientError
 
 from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
-from dbt_platform_helper.exceptions import CreateTaskTimeoutError
-from dbt_platform_helper.providers.ecs import get_ecs_task_arns
-from dbt_platform_helper.providers.secrets import get_connection_secret_arn
-from dbt_platform_helper.providers.secrets import (
-    get_postgres_connection_data_updated_with_master_secret,
-)
+from dbt_platform_helper.providers.aws import CreateTaskTimeoutException
+from dbt_platform_helper.providers.secrets import Secrets
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.messages import abort_with_error
 
@@ -17,7 +13,6 @@ from dbt_platform_helper.utils.messages import abort_with_error
 def create_addon_client_task(
     iam_client,
     ssm_client,
-    secrets_manager_client,
     subprocess,
     application: Application,
     env: str,
@@ -36,7 +31,6 @@ def create_addon_client_task(
         elif access == "admin":
             create_postgres_admin_task(
                 ssm_client,
-                secrets_manager_client,
                 subprocess,
                 application,
                 addon_name,
@@ -59,7 +53,7 @@ def create_addon_client_task(
         # We cannot check for botocore.errorfactory.NoSuchEntityException as botocore generates that class on the fly as part of errorfactory.
         # factory. Checking the error code is the recommended way of handling these exceptions.
         if ex.response.get("Error", {}).get("Code", None) != "NoSuchEntity":
-            # TODO Raise an exception to be caught at the command layer
+            # TODO When we are refactoring this, raise an exception to be caught at the command layer
             abort_with_error(
                 f"cannot obtain Role {role_name}: {ex.response.get('Error', {}).get('Message', '')}"
             )
@@ -69,7 +63,7 @@ def create_addon_client_task(
         f"--task-group-name {task_name} "
         f"{execution_role}"
         f"--image {CONDUIT_DOCKER_IMAGE_LOCATION}:{addon_type} "
-        f"--secrets CONNECTION_SECRET={get_connection_secret_arn(ssm_client,secrets_manager_client, secret_name)} "
+        f"--secrets CONNECTION_SECRET={_get_secrets_provider(application, env).get_connection_secret_arn(secret_name)} "
         "--platform-os linux "
         "--platform-arch arm64",
         shell=True,
@@ -78,7 +72,6 @@ def create_addon_client_task(
 
 def create_postgres_admin_task(
     ssm_client,
-    secrets_manager_client,
     subprocess,
     app: Application,
     addon_name: str,
@@ -95,8 +88,8 @@ def create_postgres_admin_task(
         "Parameter"
     ]["Value"]
     connection_string = json.dumps(
-        get_postgres_connection_data_updated_with_master_secret(
-            ssm_client, secrets_manager_client, read_only_secret_name, master_secret_arn
+        _get_secrets_provider(app, env).get_postgres_connection_data_updated_with_master_secret(
+            read_only_secret_name, master_secret_arn
         )
     )
 
@@ -111,6 +104,19 @@ def create_postgres_admin_task(
     )
 
 
+def _temp_until_refactor_get_ecs_task_arns(ecs_client, cluster_arn: str, task_name: str):
+    tasks = ecs_client.list_tasks(
+        cluster=cluster_arn,
+        desiredStatus="RUNNING",
+        family=f"copilot-{task_name}",
+    )
+
+    if not tasks["taskArns"]:
+        return []
+
+    return tasks["taskArns"]
+
+
 def connect_to_addon_client_task(
     ecs_client,
     subprocess,
@@ -118,13 +124,14 @@ def connect_to_addon_client_task(
     env,
     cluster_arn,
     task_name,
-    addon_client_is_running_fn=get_ecs_task_arns,
+    get_ecs_task_arns=_temp_until_refactor_get_ecs_task_arns,
 ):
     running = False
     tries = 0
     while tries < 15 and not running:
         tries += 1
-        if addon_client_is_running_fn(ecs_client, cluster_arn, task_name):
+        # Todo: Use from ECS provider when we refactor this
+        if get_ecs_task_arns(ecs_client, cluster_arn, task_name):
             subprocess.call(
                 "copilot task exec "
                 f"--app {application_name} --env {env} "
@@ -137,8 +144,18 @@ def connect_to_addon_client_task(
         time.sleep(1)
 
     if not running:
-        raise CreateTaskTimeoutError
+        raise CreateTaskTimeoutException(task_name, application_name, env)
 
 
 def _normalise_secret_name(addon_name: str) -> str:
     return addon_name.replace("-", "_").upper()
+
+
+def _get_secrets_provider(application: Application, env: str) -> Secrets:
+    # Todo: We instantiate the secrets provider here to avoid rabbit holing, but something better probably possible when we are refactoring this area
+    return Secrets(
+        application.environments[env].session.client("ssm"),
+        application.environments[env].session.client("secretsmanager"),
+        application.name,
+        env,
+    )

dbt_platform_helper/providers/ecs.py

@@ -3,77 +3,100 @@ import string
 import time
 from typing import List
 
-from dbt_platform_helper.exceptions import ECSAgentNotRunning
-from dbt_platform_helper.exceptions import NoClusterError
-
-
-# Todo: Refactor to a class, review, then perhaps do the others
-def get_cluster_arn(ecs_client, application_name: str, env: str) -> str:
-    for cluster_arn in ecs_client.list_clusters()["clusterArns"]:
-        tags_response = ecs_client.list_tags_for_resource(resourceArn=cluster_arn)
-        tags = tags_response["tags"]
-
-        app_key_found = False
-        env_key_found = False
-        cluster_key_found = False
-
-        for tag in tags:
-            if tag["key"] == "copilot-application" and tag["value"] == application_name:
-                app_key_found = True
-            if tag["key"] == "copilot-environment" and tag["value"] == env:
-                env_key_found = True
-            if tag["key"] == "aws:cloudformation:logical-id" and tag["value"] == "Cluster":
-                cluster_key_found = True
-
-        if app_key_found and env_key_found and cluster_key_found:
-            return cluster_arn
-
-    raise NoClusterError
-
-
-def get_or_create_task_name(
-    ssm_client, application_name: str, env: str, addon_name: str, parameter_name: str
-) -> str:
-    try:
-        return ssm_client.get_parameter(Name=parameter_name)["Parameter"]["Value"]
-    except ssm_client.exceptions.ParameterNotFound:
-        random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
-        return f"conduit-{application_name}-{env}-{addon_name}-{random_id}"
-
-
-def get_ecs_task_arns(ecs_client, cluster_arn: str, task_name: str):
-
-    tasks = ecs_client.list_tasks(
-        cluster=cluster_arn,
-        desiredStatus="RUNNING",
-        family=f"copilot-{task_name}",
-    )
-
-    if not tasks["taskArns"]:
-        return []
-
-    return tasks["taskArns"]
-
-
-def ecs_exec_is_available(ecs_client, cluster_arn: str, task_arns: List[str]):
-
-    current_attemps = 0
-    execute_command_agent_status = ""
-
-    while execute_command_agent_status != "RUNNING" and current_attemps < 25:
-
-        current_attemps += 1
-
-        task_details = ecs_client.describe_tasks(cluster=cluster_arn, tasks=task_arns)
-
-        managed_agents = task_details["tasks"][0]["containers"][0]["managedAgents"]
-        execute_command_agent_status = [
-            agent["lastStatus"]
-            for agent in managed_agents
-            if agent["name"] == "ExecuteCommandAgent"
-        ][0]
-
-        time.sleep(1)
-
-    if execute_command_agent_status != "RUNNING":
-        raise ECSAgentNotRunning
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ECS:
+    def __init__(self, ecs_client, ssm_client, application_name: str, env: str):
+        self.ecs_client = ecs_client
+        self.ssm_client = ssm_client
+        self.application_name = application_name
+        self.env = env
+
+    def get_cluster_arn(self) -> str:
+        """Returns the ARN of the ECS cluster for the given application and
+        environment."""
+        for cluster_arn in self.ecs_client.list_clusters()["clusterArns"]:
+            tags_response = self.ecs_client.list_tags_for_resource(resourceArn=cluster_arn)
+            tags = tags_response["tags"]
+
+            app_key_found = False
+            env_key_found = False
+            cluster_key_found = False
+
+            for tag in tags:
+                if tag["key"] == "copilot-application" and tag["value"] == self.application_name:
+                    app_key_found = True
+                if tag["key"] == "copilot-environment" and tag["value"] == self.env:
+                    env_key_found = True
+                if tag["key"] == "aws:cloudformation:logical-id" and tag["value"] == "Cluster":
+                    cluster_key_found = True
+
+            if app_key_found and env_key_found and cluster_key_found:
+                return cluster_arn
+
+        raise NoClusterException(self.application_name, self.env)
+
+    def get_or_create_task_name(self, addon_name: str, parameter_name: str) -> str:
+        """Fetches the task name from SSM or creates a new one if not found."""
+        try:
+            return self.ssm_client.get_parameter(Name=parameter_name)["Parameter"]["Value"]
+        except self.ssm_client.exceptions.ParameterNotFound:
+            random_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
+            return f"conduit-{self.application_name}-{self.env}-{addon_name}-{random_id}"
+
+    def get_ecs_task_arns(self, cluster_arn: str, task_name: str):
+        """Gets the ECS task ARNs for a given task name and cluster ARN."""
+        tasks = self.ecs_client.list_tasks(
+            cluster=cluster_arn,
+            desiredStatus="RUNNING",
+            family=f"copilot-{task_name}",
+        )
+
+        if not tasks["taskArns"]:
+            return []
+
+        return tasks["taskArns"]
+
+    def ecs_exec_is_available(self, cluster_arn: str, task_arns: List[str]):
+        """
+        Checks if the ExecuteCommandAgent is running on the specified ECS task.
+
+        Waits for up to 25 attempts, then raises ECSAgentNotRunning if still not
+        running.
+        """
+        current_attempts = 0
+        execute_command_agent_status = ""
+
+        while execute_command_agent_status != "RUNNING" and current_attempts < 25:
+            current_attempts += 1
+
+            task_details = self.ecs_client.describe_tasks(cluster=cluster_arn, tasks=task_arns)
+
+            managed_agents = task_details["tasks"][0]["containers"][0]["managedAgents"]
+            execute_command_agent_status = [
+                agent["lastStatus"]
+                for agent in managed_agents
+                if agent["name"] == "ExecuteCommandAgent"
+            ][0]
+            if execute_command_agent_status != "RUNNING":
+                time.sleep(1)
+
+        if execute_command_agent_status != "RUNNING":
+            raise ECSAgentNotRunningException
+
+
+class ECSException(PlatformException):
+    pass
+
+
+class ECSAgentNotRunningException(ECSException):
+    def __init__(self):
+        super().__init__("""ECS exec agent never reached "RUNNING" status""")
+
+
+class NoClusterException(ECSException):
+    def __init__(self, application_name: str, environment: str):
+        super().__init__(
+            f"""No ECS cluster found for "{application_name}" in "{environment}" environment."""
+        )

dbt_platform_helper/providers/load_balancers.py

@@ -1,5 +1,7 @@
 import boto3
 
+from dbt_platform_helper.platform_exception import PlatformException
+
 
 def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
     lb_client = session.client("elbv2")
@@ -16,7 +18,7 @@ def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
             load_balancer_arn = lb["ResourceArn"]
 
     if not load_balancer_arn:
-        raise LoadBalancerNotFoundError()
+        raise LoadBalancerNotFoundException()
 
     return load_balancer_arn
 
@@ -34,18 +36,22 @@ def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
         pass
 
     if not listener_arn:
-        raise ListenerNotFoundError()
+        raise ListenerNotFoundException()
 
     return listener_arn
 
 
-class LoadBalancerNotFoundError(Exception):
+class LoadBalancerException(PlatformException):
+    pass
+
+
+class LoadBalancerNotFoundException(LoadBalancerException):
     pass
 
 
-class ListenerNotFoundError(Exception):
+class ListenerNotFoundException(LoadBalancerException):
     pass
 
 
-class ListenerRuleNotFoundError(Exception):
+class ListenerRuleNotFoundException(LoadBalancerException):
     pass

dbt_platform_helper/providers/secrets.py

@@ -2,84 +2,125 @@ import json
 import urllib
 
 from dbt_platform_helper.constants import CONDUIT_ADDON_TYPES
-from dbt_platform_helper.exceptions import AddonNotFoundError
-from dbt_platform_helper.exceptions import AddonTypeMissingFromConfigError
-from dbt_platform_helper.exceptions import InvalidAddonTypeError
-from dbt_platform_helper.exceptions import ParameterNotFoundError
-from dbt_platform_helper.exceptions import SecretNotFoundError
+from dbt_platform_helper.platform_exception import PlatformException
 
 
-def get_postgres_connection_data_updated_with_master_secret(
-    ssm_client, secrets_manager_client, parameter_name, secret_arn
-):
-    response = ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)
-    parameter_value = response["Parameter"]["Value"]
+class Secrets:
+    def __init__(self, ssm_client, secrets_manager_client, application_name, env):
+        self.ssm_client = ssm_client
+        self.secrets_manager_client = secrets_manager_client
+        self.application_name = application_name
+        self.env = env
 
-    parameter_data = json.loads(parameter_value)
+    def get_postgres_connection_data_updated_with_master_secret(self, parameter_name, secret_arn):
+        response = self.ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)
+        parameter_value = response["Parameter"]["Value"]
 
-    secret_response = secrets_manager_client.get_secret_value(SecretId=secret_arn)
-    secret_value = json.loads(secret_response["SecretString"])
+        parameter_data = json.loads(parameter_value)
 
-    parameter_data["username"] = urllib.parse.quote(secret_value["username"])
-    parameter_data["password"] = urllib.parse.quote(secret_value["password"])
+        secret_response = self.secrets_manager_client.get_secret_value(SecretId=secret_arn)
+        secret_value = json.loads(secret_response["SecretString"])
 
-    return parameter_data
+        parameter_data["username"] = urllib.parse.quote(secret_value["username"])
+        parameter_data["password"] = urllib.parse.quote(secret_value["password"])
 
+        return parameter_data
 
-def get_connection_secret_arn(ssm_client, secrets_manager_client, secret_name: str) -> str:
+    def get_connection_secret_arn(self, secret_name: str) -> str:
+        try:
+            return self.ssm_client.get_parameter(Name=secret_name, WithDecryption=False)[
+                "Parameter"
+            ]["ARN"]
+        except self.ssm_client.exceptions.ParameterNotFound:
+            pass
 
-    try:
-        return ssm_client.get_parameter(Name=secret_name, WithDecryption=False)["Parameter"]["ARN"]
-    except ssm_client.exceptions.ParameterNotFound:
-        pass
+        try:
+            return self.secrets_manager_client.describe_secret(SecretId=secret_name)["ARN"]
+        except self.secrets_manager_client.exceptions.ResourceNotFoundException:
+            pass
 
-    try:
-        return secrets_manager_client.describe_secret(SecretId=secret_name)["ARN"]
-    except secrets_manager_client.exceptions.ResourceNotFoundException:
-        pass
+        raise SecretNotFoundException(secret_name)
 
-    raise SecretNotFoundError(secret_name)
+    # Todo: This probably does not belong in the secrets provider. When it moves, take the Todoed exceptions from below
+    def get_addon_type(self, addon_name: str) -> str:
+        addon_type = None
+        try:
+            addon_config = json.loads(
+                self.ssm_client.get_parameter(
+                    Name=f"/copilot/applications/{self.application_name}/environments/{self.env}/addons"
+                )["Parameter"]["Value"]
+            )
+        except self.ssm_client.exceptions.ParameterNotFound:
+            raise ParameterNotFoundException(self.application_name, self.env)
 
+        if addon_name not in addon_config.keys():
+            raise AddonNotFoundException(addon_name)
+
+        for name, config in addon_config.items():
+            if name == addon_name:
+                if not config.get("type"):
+                    raise AddonTypeMissingFromConfigException(addon_name)
+                addon_type = config["type"]
+
+        if not addon_type or addon_type not in CONDUIT_ADDON_TYPES:
+            raise InvalidAddonTypeException(addon_type)
+
+        if "postgres" in addon_type:
+            addon_type = "postgres"
+
+        return addon_type
+
+    def get_parameter_name(self, addon_type: str, addon_name: str, access: str) -> str:
+        if addon_type == "postgres":
+            return f"/copilot/{self.application_name}/{self.env}/conduits/{self._normalise_secret_name(addon_name)}_{access.upper()}"
+        elif addon_type == "redis" or addon_type == "opensearch":
+            return f"/copilot/{self.application_name}/{self.env}/conduits/{self._normalise_secret_name(addon_name)}_ENDPOINT"
+        else:
+            return f"/copilot/{self.application_name}/{self.env}/conduits/{self._normalise_secret_name(addon_name)}"
+
+    def _normalise_secret_name(self, addon_name: str) -> str:
+        return addon_name.replace("-", "_").upper()
 
-def get_addon_type(ssm_client, application_name: str, env: str, addon_name: str) -> str:
-    addon_type = None
-    try:
-        addon_config = json.loads(
-            ssm_client.get_parameter(
-                Name=f"/copilot/applications/{application_name}/environments/{env}/addons"
-            )["Parameter"]["Value"]
-        )
-    except ssm_client.exceptions.ParameterNotFound:
-        raise ParameterNotFoundError
 
-    if addon_name not in addon_config.keys():
-        raise AddonNotFoundError
+# Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
+class AddonException(PlatformException):
+    pass
 
-    for name, config in addon_config.items():
-        if name == addon_name:
-            if not config.get("type"):
-                raise AddonTypeMissingFromConfigError()
-            addon_type = config["type"]
 
-    if not addon_type or addon_type not in CONDUIT_ADDON_TYPES:
-        raise InvalidAddonTypeError(addon_type)
+# Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
+class AddonNotFoundException(AddonException):
+    def __init__(self, addon_name: str):
+        super().__init__(f"""Addon "{addon_name}" does not exist.""")
 
-    if "postgres" in addon_type:
-        addon_type = "postgres"
 
-    return addon_type
+# Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
+class AddonTypeMissingFromConfigException(AddonException):
+    def __init__(self, addon_name: str):
+        super().__init__(
+            f"""The configuration for the addon {addon_name}, is misconfigured and missing the addon type."""
+        )
+
+
+# Todo: This probably does not belong in the secrets provider. Move it when we find a better home for get_addon_type()
+class InvalidAddonTypeException(AddonException):
+    def __init__(self, addon_type):
+        self.addon_type = addon_type
+        super().__init__(
+            f"""Addon type "{self.addon_type}" is not supported, we support: {", ".join(CONDUIT_ADDON_TYPES)}."""
+        )
 
 
-def get_parameter_name(
-    application_name: str, env: str, addon_type: str, addon_name: str, access: str
-) -> str:
-    if addon_type == "postgres":
-        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}_{access.upper()}"
-    elif addon_type == "redis" or addon_type == "opensearch":
-        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}_ENDPOINT"
-    else:
-        return f"/copilot/{application_name}/{env}/conduits/{_normalise_secret_name(addon_name)}"
+class SecretException(PlatformException):
+    pass
+
+
+class ParameterNotFoundException(SecretException):
+    def __init__(self, application_name: str, environment: str):
+        super().__init__(
+            f"""No parameter called "/copilot/applications/{application_name}/environments/{environment}/addons". Try deploying the "{application_name}" "{environment}" environment."""
+        )
 
 
-def _normalise_secret_name(addon_name: str) -> str:
-    return addon_name.replace("-", "_").upper()
+class SecretNotFoundException(SecretException):
+    def __init__(self, secret_name: str):
+        super().__init__(f"""No secret called "{secret_name}".""")

dbt_platform_helper/providers/validation.py

@@ -0,0 +1,19 @@
+from dbt_platform_helper.platform_exception import PlatformException
+
+
+class ValidationException(PlatformException):
+    pass
+
+
+class IncompatibleMajorVersionException(ValidationException):
+    def __init__(self, app_version: str, check_version: str):
+        super().__init__()
+        self.app_version = app_version
+        self.check_version = check_version
+
+
+class IncompatibleMinorVersionException(ValidationException):
+    def __init__(self, app_version: str, check_version: str):
+        super().__init__()
+        self.app_version = app_version
+        self.check_version = check_version

dbt_platform_helper/utils/application.py

@@ -1,4 +1,5 @@
 import json
+import os
 import re
 from pathlib import Path
 from typing import Dict
@@ -8,7 +9,7 @@ import yaml
 from boto3 import Session
 from yaml.parser import ParserError
 
-from dbt_platform_helper.exceptions import ApplicationNotFoundError
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_profile_name_from_account_id
 from dbt_platform_helper.utils.aws import get_ssm_secrets
@@ -80,7 +81,7 @@ def load_application(app: str = None, default_session: Session = None) -> Applic
             WithDecryption=False,
         )
     except ssm_client.exceptions.ParameterNotFound:
-        raise ApplicationNotFoundError
+        raise ApplicationNotFoundException(app)
 
     path = f"/copilot/applications/{application.name}/environments"
     secrets = get_ssm_secrets(app, None, current_session, path)
@@ -135,3 +136,14 @@ def get_application_name():
         abort_with_error("Cannot get application name. No copilot/.workspace file found")
 
     return app_name
+
+
+class ApplicationException(PlatformException):
+    pass
+
+
+class ApplicationNotFoundException(ApplicationException):
+    def __init__(self, application_name: str):
+        super().__init__(
+            f"""The account "{os.environ.get("AWS_PROFILE")}" does not contain the application "{application_name}"; ensure you have set the environment variable "AWS_PROFILE" correctly."""
+        )

dbt_platform_helper/utils/arn_parser.py

@@ -1,4 +1,4 @@
-from dbt_platform_helper.exceptions import ValidationException
+from dbt_platform_helper.providers.validation import ValidationException
 
 
 class ARN: