dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/domain/codebase.py

@@ -3,35 +3,43 @@ import stat
 import subprocess
 from collections.abc import Callable
 from pathlib import Path
+from typing import Tuple
 
 import requests
 import yaml
 from boto3 import Session
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.ecr import ECRProvider
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
 from dbt_platform_helper.utils.application import Application
-from dbt_platform_helper.utils.application import ApplicationException
+from dbt_platform_helper.utils.application import (
+    ApplicationEnvironmentNotFoundException,
+)
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import check_image_exists
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.aws import get_build_url_from_arn
 from dbt_platform_helper.utils.aws import get_build_url_from_pipeline_execution_id
+from dbt_platform_helper.utils.aws import get_image_build_project
+from dbt_platform_helper.utils.aws import get_manual_release_pipeline
 from dbt_platform_helper.utils.aws import list_latest_images
 from dbt_platform_helper.utils.aws import start_build_extraction
 from dbt_platform_helper.utils.aws import start_pipeline_and_return_execution_id
-from dbt_platform_helper.utils.git import check_if_commit_exists
 from dbt_platform_helper.utils.template import setup_templates
 
 
 class Codebase:
     def __init__(
         self,
+        parameter_provider: ParameterStore,
         io: ClickIOProvider = ClickIOProvider(),
         load_application: Callable[[str], Application] = load_application,
         get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
-        check_image_exists: Callable[[str], str] = check_image_exists,
+        ecr_provider: ECRProvider = ECRProvider(),
+        get_image_build_project: Callable[[str], str] = get_image_build_project,
+        get_manual_release_pipeline: Callable[[str], str] = get_manual_release_pipeline,
         get_build_url_from_arn: Callable[[str], str] = get_build_url_from_arn,
         get_build_url_from_pipeline_execution_id: Callable[
             [str], str
@@ -41,19 +49,20 @@ class Codebase:
         start_pipeline_and_return_execution_id: Callable[
             [str], str
         ] = start_pipeline_and_return_execution_id,
-        check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
         run_subprocess: Callable[[str], str] = subprocess.run,
     ):
+        self.parameter_provider = parameter_provider
         self.io = io
         self.load_application = load_application
         self.get_aws_session_or_abort = get_aws_session_or_abort
-        self.check_image_exists = check_image_exists
+        self.ecr_provider = ecr_provider
+        self.get_image_build_project = get_image_build_project
+        self.get_manual_release_pipeline = get_manual_release_pipeline
         self.get_build_url_from_arn = get_build_url_from_arn
         self.get_build_url_from_pipeline_execution_id = get_build_url_from_pipeline_execution_id
         self.list_latest_images = list_latest_images
         self.start_build_extraction = start_build_extraction
         self.start_pipeline_and_return_execution_id = start_pipeline_and_return_execution_id
-        self.check_if_commit_exists = check_if_commit_exists
         self.run_subprocess = run_subprocess
 
     def prepare(self):
@@ -121,15 +130,14 @@ class Codebase:
         session = self.get_aws_session_or_abort()
         self.load_application(app, default_session=session)
 
-        self.check_if_commit_exists(commit)
-
         codebuild_client = session.client("codebuild")
+        project_name = self.get_image_build_project(codebuild_client, app, codebase)
         build_url = self.__start_build_with_confirmation(
             codebuild_client,
             self.get_build_url_from_arn,
             f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
             {
-                "projectName": f"{app}-{codebase}-codebase-pipeline-image-build",
+                "projectName": project_name,
                 "artifactsOverride": {"type": "NO_ARTIFACTS"},
                 "sourceVersion": commit,
             },
@@ -142,30 +150,57 @@ class Codebase:
 
         raise ApplicationDeploymentNotTriggered(codebase)
 
-    def deploy(self, app, env, codebase, commit):
+    def deploy(
+        self,
+        app: str,
+        env: str,
+        codebase: str,
+        commit: str = None,
+        tag: str = None,
+        branch: str = None,
+    ):
         """Trigger a CodePipeline pipeline based deployment."""
-        session = self.get_aws_session_or_abort()
 
-        application = self.load_application(app, default_session=session)
-        if not application.environments.get(env):
-            raise ApplicationEnvironmentNotFoundException(env)
+        self._validate_reference_flags(commit, tag, branch)
+
+        application, session = self._populate_application_values(app, env)
 
-        self.check_image_exists(session, application, codebase, commit)
+        image_ref = None
+        if commit:
+            self._validate_sha_length(commit)
+            image_ref = f"commit-{commit}"
+        elif tag:
+            image_ref = f"tag-{tag}"
+        elif branch:
+            image_ref = f"branch-{branch}"
+
+        image_ref = self.ecr_provider.get_commit_tag_for_reference(
+            application.name, codebase, image_ref
+        )
 
-        pipeline_name = f"{app}-{codebase}-manual-release-pipeline"
         codepipeline_client = session.client("codepipeline")
+        pipeline_name = self.get_manual_release_pipeline(codepipeline_client, app, codebase)
+
+        corresponding_to = ""
+        if tag:
+            corresponding_to = f"(corresponding to tag {tag}) "
+        elif branch:
+            corresponding_to = f"(corresponding to branch {branch}) "
+
+        confirmation_message = f'\nYou are about to deploy "{app}" for "{codebase}" with image reference "{image_ref}" {corresponding_to}to the "{env}" environment using the "{pipeline_name}" deployment pipeline. Do you want to continue?'
+        build_options = {
+            "name": pipeline_name,
+            "variables": [
+                {"name": "ENVIRONMENT", "value": env},
+                {"name": "IMAGE_TAG", "value": image_ref},
+            ],
+        }
 
         build_url = self.__start_pipeline_execution_with_confirmation(
             codepipeline_client,
             self.get_build_url_from_pipeline_execution_id,
-            f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment using the "{pipeline_name}" deployment pipeline. Do you want to continue?',
-            {
-                "name": pipeline_name,
-                "variables": [
-                    {"name": "ENVIRONMENT", "value": env},
-                    {"name": "IMAGE_TAG", "value": f"commit-{commit}"},
-                ],
-            },
+            confirmation_message,
+            build_options,
         )
 
         if build_url:
@@ -176,13 +211,31 @@ class Codebase:
 
         raise ApplicationDeploymentNotTriggered(codebase)
 
+    def _validate_reference_flags(self, commit: str, tag: str, branch: str):
+        provided = [ref for ref in [commit, tag, branch] if ref]
+
+        if len(provided) == 0:
+            self.io.abort_with_error(
+                "To deploy, you must provide one of the options --commit, --tag or --branch."
+            )
+        elif len(provided) > 1:
+            self.io.abort_with_error(
+                "You have provided more than one of the --tag, --branch and --commit options but these are mutually exclusive. Please provide only one of these options."
+            )
+
+    def _populate_application_values(self, app: str, env: str) -> Tuple[Application, Session]:
+        session = self.get_aws_session_or_abort()
+        application = self.load_application(app, default_session=session)
+        if not application.environments.get(env):
+            raise ApplicationEnvironmentNotFoundException(application.name, env)
+        return application, session
+
     def list(self, app: str, with_images: bool):
         """List available codebases for the application."""
         session = self.get_aws_session_or_abort()
         application = self.load_application(app, session)
-        ssm_client = session.client("ssm")
         ecr_client = session.client("ecr")
-        codebases = self.__get_codebases(application, ssm_client)
+        codebases = self.__get_codebases(application, session.client("ssm"))
 
         self.io.info("The following codebases are available:")
 
@@ -199,11 +252,9 @@ class Codebase:
         self.io.info("")
 
     def __get_codebases(self, application, ssm_client):
-        parameters = ssm_client.get_parameters_by_path(
-            Path=f"/copilot/applications/{application.name}/codebases",
-            Recursive=True,
-        )["Parameters"]
-
+        parameters = self.parameter_provider.get_ssm_parameters_by_path(
+            f"/copilot/applications/{application.name}/codebases"
+        )
         codebases = [json.loads(p["Value"]) for p in parameters]
 
         if not codebases:
@@ -236,19 +287,18 @@ class Codebase:
             return get_build_url_from_pipeline_execution_id(execution_id, build_options["name"])
         return None
 
+    def _validate_sha_length(self, commit):
+        if len(commit) < 7:
+            self.io.abort_with_error(
+                "Your commit reference is too short. Commit sha hashes specified by '--commit' must be at least 7 characters long."
+            )
+
 
 class ApplicationDeploymentNotTriggered(PlatformException):
     def __init__(self, codebase: str):
         super().__init__(f"""Your deployment for {codebase} was not triggered.""")
 
 
-class ApplicationEnvironmentNotFoundException(ApplicationException):
-    def __init__(self, environment: str):
-        super().__init__(
-            f"""The environment "{environment}" either does not exist or has not been deployed."""
-        )
-
-
 class NotInCodeBaseRepositoryException(PlatformException):
     def __init__(self):
         super().__init__(

dbt_platform_helper/domain/conduit.py

@@ -1,119 +1,378 @@
-import subprocess
+import json
+from abc import ABC
+from abc import abstractmethod
+from typing import Callable
+from typing import Optional
 
 from dbt_platform_helper.providers.cloudformation import CloudFormation
+from dbt_platform_helper.providers.copilot import _normalise_secret_name
 from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
 from dbt_platform_helper.providers.copilot import create_addon_client_task
+from dbt_platform_helper.providers.copilot import get_postgres_admin_connection_string
 from dbt_platform_helper.providers.ecs import ECS
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.secrets import Secrets
+from dbt_platform_helper.providers.vpc import VpcProvider
 from dbt_platform_helper.utils.application import Application
 
 
-class Conduit:
+class ConduitECSStrategy(ABC):
+    @abstractmethod
+    def get_data(self):
+        pass
+
+    @abstractmethod
+    def start_task(self, data_context: dict):
+        pass
+
+    @abstractmethod
+    def exec_task(self, data_context: dict):
+        pass
+
+
+class TerraformConduitStrategy(ConduitECSStrategy):
     def __init__(
         self,
+        clients,
+        ecs_provider: ECS,
         application: Application,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+        vpc_provider: Callable,
+        get_postgres_admin_connection_string: Callable,
+    ):
+        self.clients = clients
+        self.ecs_provider = ecs_provider
+        self.io = io
+        self.vpc_provider = vpc_provider
+        self.access = access
+        self.addon_name = addon_name
+        self.addon_type = addon_type
+        self.application = application
+        self.env = env
+        self.get_postgres_admin_connection_string = get_postgres_admin_connection_string
+
+    def get_data(self):
+        self.io.info("Starting conduit in Terraform mode.")
+        try:
+            cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
+                f"{self.application.name}-{self.env}-cluster"
+            )
+        except:
+            cluster_arn = self.ecs_provider.get_cluster_arn_by_name(
+                f"{self.application.name}-{self.env}"
+            )
+        return {
+            "cluster_arn": cluster_arn,
+            "task_def_family": self._generate_container_name(),
+            "vpc_name": self._resolve_vpc_name(),
+            "addon_type": self.addon_type,
+            "access": self.access,
+        }
+
+    def start_task(self, data_context: dict):
+
+        environments = self.application.environments
+        environment = environments.get(self.env)
+        env_session = environment.session
+
+        vpc_provider = self.vpc_provider(env_session)
+        vpc_config = vpc_provider.get_vpc(
+            self.application.name,
+            self.env,
+            data_context["vpc_name"],
+        )
+
+        postgres_admin_env_vars = None
+        if data_context["addon_type"] == "postgres" and data_context["access"] == "admin":
+            postgres_admin_env_vars = [
+                {
+                    "name": "CONNECTION_SECRET",
+                    "value": self.get_postgres_admin_connection_string(
+                        self.clients.get("ssm"),
+                        f"/copilot/{self.application.name}/{self.env}/secrets/{_normalise_secret_name(self.addon_name)}",
+                        self.application,
+                        self.env,
+                        self.addon_name,
+                    ),
+                },
+            ]
+
+        cluster_name = data_context["cluster_arn"].split("/")[-1]
+
+        self.ecs_provider.start_ecs_task(
+            cluster_name,
+            self._generate_container_name(),
+            data_context["task_def_family"],
+            vpc_config,
+            postgres_admin_env_vars,
+        )
+
+    def exec_task(self, data_context: dict):
+        self.ecs_provider.exec_task(data_context["cluster_arn"], data_context["task_arns"][0])
+
+    def _generate_container_name(self):
+        return f"conduit-{self.addon_type}-{self.access}-{self.application.name}-{self.env}-{self.addon_name}"
+
+    def _resolve_vpc_name(self):
+        ssm_client = self.clients["ssm"]
+        parameter_key = f"/platform/applications/{self.application.name}/environments/{self.env}"
+
+        try:
+            response = ssm_client.get_parameter(Name=parameter_key)["Parameter"]["Value"]
+            return json.loads(response)["vpc_name"]
+        except ssm_client.exceptions.ParameterNotFound:
+            self.io.abort_with_error(
+                f"Could not find AWS SSM parameter {parameter_key}. Please ensure your environment Terraform is up to date."
+            )
+        except KeyError:
+            self.io.abort_with_error(
+                f"The parameter {parameter_key} exists but does not contain the 'vpc_name' field. Please ensure your environment Terraform is up to date."
+            )
+
+
+class CopilotConduitStrategy(ConduitECSStrategy):
+    def __init__(
+        self,
+        clients,
+        ecs_provider: ECS,
         secrets_provider: Secrets,
         cloudformation_provider: CloudFormation,
-        ecs_provider: ECS,
-        io: ClickIOProvider = ClickIOProvider(),
-        subprocess: subprocess = subprocess,
-        connect_to_addon_client_task=connect_to_addon_client_task,
-        create_addon_client_task=create_addon_client_task,
+        application: Application,
+        addon_name: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+        connect_to_addon_client_task: Callable,
+        create_addon_client_task: Callable,
     ):
-
-        self.application = application
-        self.secrets_provider = secrets_provider
+        self.clients = clients
         self.cloudformation_provider = cloudformation_provider
         self.ecs_provider = ecs_provider
-        self.subprocess = subprocess
+        self.secrets_provider = secrets_provider
+
         self.io = io
+        self.access = access
+        self.addon_name = addon_name
+        self.application = application
+        self.env = env
         self.connect_to_addon_client_task = connect_to_addon_client_task
         self.create_addon_client_task = create_addon_client_task
 
-    def start(self, env: str, addon_name: str, access: str = "read"):
-        clients = self._initialise_clients(env)
-        addon_type, cluster_arn, parameter_name, task_name = self._get_addon_details(
-            addon_name, access
+    def get_data(self):
+
+        addon_type = self.secrets_provider.get_addon_type(self.addon_name)
+        parameter_name = self.secrets_provider.get_parameter_name(
+            addon_type, self.addon_name, self.access
         )
+        task_name = self.ecs_provider.get_or_create_task_name(self.addon_name, parameter_name)
 
-        self.io.info(f"Checking if a conduit task is already running for {addon_type}")
-        task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
-        if not task_arns:
-            self.io.info("Creating conduit task")
-            self.create_addon_client_task(
-                clients["iam"],
-                clients["ssm"],
-                self.subprocess,
-                self.application,
-                env,
-                addon_type,
+        return {
+            "cluster_arn": self.ecs_provider.get_cluster_arn_by_copilot_tag(),
+            "addon_type": addon_type,
+            "task_def_family": f"copilot-{task_name}",
+            "parameter_name": parameter_name,
+            "task_name": task_name,
+        }
+
+    def start_task(self, data_context: dict):
+        self.create_addon_client_task(
+            self.clients["iam"],
+            self.clients["ssm"],
+            self.application,
+            self.env,
+            data_context["addon_type"],
+            self.addon_name,
+            data_context["task_name"],
+            self.access,
+        )
+
+        self.io.info("Updating conduit task")
+        self.cloudformation_provider.add_stack_delete_policy_to_task_role(data_context["task_name"])
+        stack_name = self.cloudformation_provider.update_conduit_stack_resources(
+            self.application.name,
+            self.env,
+            data_context["addon_type"],
+            self.addon_name,
+            data_context["task_name"],
+            data_context["parameter_name"],
+            self.access,
+        )
+        self.io.info("Waiting for conduit task update to complete...")
+        self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
+            "stack_update_complete", stack_name
+        )
+
+    def exec_task(self, data_context: dict):
+        self.connect_to_addon_client_task(
+            self.clients["ecs"],
+            self.application.name,
+            self.env,
+            data_context["cluster_arn"],
+            data_context["task_name"],
+        )
+
+
+class ConduitStrategyFactory:
+
+    @staticmethod
+    def detect_mode(
+        ecs_client,
+        application,
+        environment,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        io: ClickIOProvider,
+    ) -> str:
+        """Detect if Terraform-based conduit task definitions are present,
+        otherwise default to Copilot mode."""
+        paginator = ecs_client.get_paginator("list_task_definitions")
+        prefix = f"conduit-{addon_type}-{access}-{application}-{environment}-{addon_name}"
+
+        for page in paginator.paginate():
+            for arn in page["taskDefinitionArns"]:
+                if arn.split("/")[-1].startswith(prefix):
+                    return "terraform"
+
+        io.info("Defaulting to copilot mode.")
+        return "copilot"
+
+    @staticmethod
+    def create_strategy(
+        mode: str,
+        clients,
+        ecs_provider: ECS,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        application: Application,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+    ):
+
+        if mode == "terraform":
+            return TerraformConduitStrategy(
+                clients,
+                ecs_provider,
+                application,
                 addon_name,
-                task_name,
+                addon_type,
                 access,
-            )
-
-            self.io.info("Updating conduit task")
-            self._update_stack_resources(
-                self.application.name,
                 env,
-                addon_type,
+                io,
+                vpc_provider=VpcProvider,
+                get_postgres_admin_connection_string=get_postgres_admin_connection_string,
+            )
+        else:
+            return CopilotConduitStrategy(
+                clients,
+                ecs_provider,
+                secrets_provider,
+                cloudformation_provider,
+                application,
                 addon_name,
-                task_name,
-                parameter_name,
                 access,
+                env,
+                io,
+                connect_to_addon_client_task=connect_to_addon_client_task,
+                create_addon_client_task=create_addon_client_task,
             )
 
-            task_arns = self.ecs_provider.get_ecs_task_arns(cluster_arn, task_name)
 
-        else:
-            self.io.info("Conduit task already running")
+class Conduit:
+    def __init__(
+        self,
+        application: Application,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        ecs_provider: ECS,
+        io: ClickIOProvider = ClickIOProvider(),
+        vpc_provider=VpcProvider,
+        strategy_factory: Optional[ConduitStrategyFactory] = None,
+    ):
 
-        self.io.info(f"Checking if exec is available for conduit task...")
+        self.application = application
+        self.secrets_provider = secrets_provider
+        self.cloudformation_provider = cloudformation_provider
+        self.ecs_provider = ecs_provider
+        self.io = io
+        self.vpc_provider = vpc_provider
+        self.strategy_factory = strategy_factory or ConduitStrategyFactory()
+
+    def start(self, env: str, addon_name: str, access: str = "read"):
+        self.clients = self._initialise_clients(env)
+        addon_type = self.secrets_provider.get_addon_type(addon_name)
 
-        self.ecs_provider.ecs_exec_is_available(cluster_arn, task_arns)
+        if (addon_type == "opensearch" or addon_type == "redis") and (access != "read"):
+            access = "read"
 
-        self.io.info("Connecting to conduit task")
-        self.connect_to_addon_client_task(
-            clients["ecs"], self.subprocess, self.application.name, env, cluster_arn, task_name
+        mode = self.strategy_factory.detect_mode(
+            self.clients.get("ecs"),
+            self.application.name,
+            env,
+            addon_name,
+            addon_type,
+            access,
+            self.io,
+        )
+
+        strategy = self.strategy_factory.create_strategy(
+            mode=mode,
+            clients=self.clients,
+            ecs_provider=self.ecs_provider,
+            secrets_provider=self.secrets_provider,
+            cloudformation_provider=self.cloudformation_provider,
+            application=self.application,
+            addon_name=addon_name,
+            addon_type=addon_type,
+            access=access,
+            env=env,
+            io=self.io,
+        )
+
+        data_context = strategy.get_data()
+
+        data_context["task_arns"] = self.ecs_provider.get_ecs_task_arns(
+            cluster=data_context["cluster_arn"], task_def_family=data_context["task_def_family"]
+        )
+
+        info_log = (
+            f"Checking if a conduit ECS task is already running for:\n"
+            f"  Addon Name : {addon_name}\n"
+            f"  Addon Type : {addon_type}"
         )
 
+        if addon_type == "postgres":
+            info_log += f"\n  Access Level : {access}"
+
+        self.io.info(info_log)
+
+        if not data_context["task_arns"]:
+            self.io.info("Creating conduit ECS task...")
+            strategy.start_task(data_context)
+            data_context["task_arns"] = self.ecs_provider.wait_for_task_to_register(
+                data_context["cluster_arn"], data_context["task_def_family"]
+            )
+        else:
+            self.io.info(f"Found a task already running: {data_context['task_arns'][0]}")
+
+        self.io.info(f"Waiting for ECS Exec agent to become available on the conduit task...")
+        self.ecs_provider.ecs_exec_is_available(
+            data_context["cluster_arn"], data_context["task_arns"]
+        )
+
+        self.io.info("Connecting to conduit task...")
+        strategy.exec_task(data_context)
+
     def _initialise_clients(self, env):
         return {
             "ecs": self.application.environments[env].session.client("ecs"),
             "iam": self.application.environments[env].session.client("iam"),
             "ssm": self.application.environments[env].session.client("ssm"),
         }
-
-    def _get_addon_details(self, addon_name, access):
-        addon_type = self.secrets_provider.get_addon_type(addon_name)
-        cluster_arn = self.ecs_provider.get_cluster_arn()
-        parameter_name = self.secrets_provider.get_parameter_name(addon_type, addon_name, access)
-        task_name = self.ecs_provider.get_or_create_task_name(addon_name, parameter_name)
-
-        return addon_type, cluster_arn, parameter_name, task_name
-
-    def _update_stack_resources(
-        self,
-        app_name,
-        env,
-        addon_type,
-        addon_name,
-        task_name,
-        parameter_name,
-        access,
-    ):
-        self.cloudformation_provider.add_stack_delete_policy_to_task_role(task_name)
-        stack_name = self.cloudformation_provider.update_conduit_stack_resources(
-            app_name,
-            env,
-            addon_type,
-            addon_name,
-            task_name,
-            parameter_name,
-            access,
-        )
-        self.io.info("Waiting for conduit task update to complete...")
-        self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
-            "stack_update_complete", stack_name
-        )