dbt-platform-helper 15.1.0__tar.gz → 15.2.1__tar.gz

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 15.1.0
+Version: 15.2.1
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/domain/codebase.py

@@ -167,13 +167,16 @@ class Codebase:
 
         image_ref = None
         if commit:
-            image_ref = f"commit-{commit[0:7]}"
+            self._validate_sha_length(commit)
+            image_ref = f"commit-{commit}"
         elif tag:
             image_ref = f"tag-{tag}"
         elif branch:
             image_ref = f"branch-{branch}"
-        image_details = self.ecr_provider.get_image_details(application, codebase, image_ref)
-        image_ref = self.ecr_provider.find_commit_tag(image_details, image_ref)
+
+        image_ref = self.ecr_provider.get_commit_tag_for_reference(
+            application.name, codebase, image_ref
+        )
 
         codepipeline_client = session.client("codepipeline")
         pipeline_name = self.get_manual_release_pipeline(codepipeline_client, app, codebase)
@@ -284,6 +287,12 @@ class Codebase:
             return get_build_url_from_pipeline_execution_id(execution_id, build_options["name"])
         return None
 
+    def _validate_sha_length(self, commit):
+        if len(commit) < 7:
+            self.io.abort_with_error(
+                "Your commit reference is too short. Commit sha hashes specified by '--commit' must be at least 7 characters long."
+            )
+
 
 class ApplicationDeploymentNotTriggered(PlatformException):
     def __init__(self, codebase: str):

dbt_platform_helper-15.2.1/dbt_platform_helper/domain/conduit.py

@@ -0,0 +1,365 @@
+from abc import ABC
+from abc import abstractmethod
+from typing import Callable
+from typing import Optional
+
+from dbt_platform_helper.providers.cloudformation import CloudFormation
+from dbt_platform_helper.providers.copilot import _normalise_secret_name
+from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
+from dbt_platform_helper.providers.copilot import create_addon_client_task
+from dbt_platform_helper.providers.copilot import get_postgres_admin_connection_string
+from dbt_platform_helper.providers.ecs import ECS
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.secrets import Secrets
+from dbt_platform_helper.providers.vpc import VpcProvider
+from dbt_platform_helper.utils.application import Application
+
+
+class ConduitECSStrategy(ABC):
+    @abstractmethod
+    def get_data(self):
+        pass
+
+    @abstractmethod
+    def start_task(self, data_context: dict):
+        pass
+
+    @abstractmethod
+    def exec_task(self, data_context: dict):
+        pass
+
+
+class TerraformConduitStrategy(ConduitECSStrategy):
+    def __init__(
+        self,
+        clients,
+        ecs_provider: ECS,
+        application: Application,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+        vpc_provider: Callable,
+        get_postgres_admin_connection_string: Callable,
+    ):
+        self.clients = clients
+        self.ecs_provider = ecs_provider
+        self.io = io
+        self.vpc_provider = vpc_provider
+        self.access = access
+        self.addon_name = addon_name
+        self.addon_type = addon_type
+        self.application = application
+        self.env = env
+        self.get_postgres_admin_connection_string = get_postgres_admin_connection_string
+
+    def get_data(self):
+        self.io.info("Starting conduit in Terraform mode.")
+        return {
+            "cluster_arn": self.ecs_provider.get_cluster_arn_by_name(
+                f"{self.application.name}-{self.env}"
+            ),
+            "task_def_family": self._generate_container_name(),
+            "vpc_name": self._resolve_vpc_name(),
+            "addon_type": self.addon_type,
+            "access": self.access,
+        }
+
+    def start_task(self, data_context: dict):
+
+        environments = self.application.environments
+        environment = environments.get(self.env)
+        env_session = environment.session
+
+        vpc_provider = self.vpc_provider(env_session)
+        vpc_config = vpc_provider.get_vpc(
+            self.application.name,
+            self.env,
+            data_context["vpc_name"],
+        )
+
+        postgres_admin_env_vars = None
+        if data_context["addon_type"] == "postgres" and data_context["access"] == "admin":
+            postgres_admin_env_vars = [
+                {
+                    "name": "CONNECTION_SECRET",
+                    "value": self.get_postgres_admin_connection_string(
+                        self.clients.get("ssm"),
+                        f"/copilot/{self.application.name}/{self.env}/secrets/{_normalise_secret_name(self.addon_name)}",
+                        self.application,
+                        self.env,
+                        self.addon_name,
+                    ),
+                },
+            ]
+
+        self.ecs_provider.start_ecs_task(
+            f"{self.application.name}-{self.env}",
+            self._generate_container_name(),
+            data_context["task_def_family"],
+            vpc_config,
+            postgres_admin_env_vars,
+        )
+
+    def exec_task(self, data_context: dict):
+        self.ecs_provider.exec_task(data_context["cluster_arn"], data_context["task_arns"][0])
+
+    def _generate_container_name(self):
+        return f"conduit-{self.addon_type}-{self.access}-{self.application.name}-{self.env}-{self.addon_name}"
+
+    def _resolve_vpc_name(self):
+        ssm_client = self.clients["ssm"]
+        parameter_key = f"/conduit/{self.application.name}/{self.env}/{_normalise_secret_name(self.addon_name)}_VPC_NAME"
+
+        try:
+            response = ssm_client.get_parameter(Name=parameter_key)
+            return response["Parameter"]["Value"]
+        except ssm_client.exceptions.ParameterNotFound:
+            self.io.abort_with_error(
+                f"Could not find VPC name for {self.addon_name}. Missing SSM param: {parameter_key}"
+            )
+
+
+class CopilotConduitStrategy(ConduitECSStrategy):
+    def __init__(
+        self,
+        clients,
+        ecs_provider: ECS,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        application: Application,
+        addon_name: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+        connect_to_addon_client_task: Callable,
+        create_addon_client_task: Callable,
+    ):
+        self.clients = clients
+        self.cloudformation_provider = cloudformation_provider
+        self.ecs_provider = ecs_provider
+        self.secrets_provider = secrets_provider
+
+        self.io = io
+        self.access = access
+        self.addon_name = addon_name
+        self.application = application
+        self.env = env
+        self.connect_to_addon_client_task = connect_to_addon_client_task
+        self.create_addon_client_task = create_addon_client_task
+
+    def get_data(self):
+
+        addon_type = self.secrets_provider.get_addon_type(self.addon_name)
+        parameter_name = self.secrets_provider.get_parameter_name(
+            addon_type, self.addon_name, self.access
+        )
+        task_name = self.ecs_provider.get_or_create_task_name(self.addon_name, parameter_name)
+
+        return {
+            "cluster_arn": self.ecs_provider.get_cluster_arn_by_copilot_tag(),
+            "addon_type": addon_type,
+            "task_def_family": f"copilot-{task_name}",
+            "parameter_name": parameter_name,
+            "task_name": task_name,
+        }
+
+    def start_task(self, data_context: dict):
+        self.create_addon_client_task(
+            self.clients["iam"],
+            self.clients["ssm"],
+            self.application,
+            self.env,
+            data_context["addon_type"],
+            self.addon_name,
+            data_context["task_name"],
+            self.access,
+        )
+
+        self.io.info("Updating conduit task")
+        self.cloudformation_provider.add_stack_delete_policy_to_task_role(data_context["task_name"])
+        stack_name = self.cloudformation_provider.update_conduit_stack_resources(
+            self.application.name,
+            self.env,
+            data_context["addon_type"],
+            self.addon_name,
+            data_context["task_name"],
+            data_context["parameter_name"],
+            self.access,
+        )
+        self.io.info("Waiting for conduit task update to complete...")
+        self.cloudformation_provider.wait_for_cloudformation_to_reach_status(
+            "stack_update_complete", stack_name
+        )
+
+    def exec_task(self, data_context: dict):
+        self.connect_to_addon_client_task(
+            self.clients["ecs"],
+            self.application.name,
+            self.env,
+            data_context["cluster_arn"],
+            data_context["task_name"],
+        )
+
+
+class ConduitStrategyFactory:
+
+    @staticmethod
+    def detect_mode(
+        ecs_client,
+        application,
+        environment,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        io: ClickIOProvider,
+    ) -> str:
+        """Detect if Terraform-based conduit task definitions are present,
+        otherwise default to Copilot mode."""
+        paginator = ecs_client.get_paginator("list_task_definitions")
+        prefix = f"conduit-{addon_type}-{access}-{application}-{environment}-{addon_name}"
+
+        for page in paginator.paginate():
+            for arn in page["taskDefinitionArns"]:
+                if arn.split("/")[-1].startswith(prefix):
+                    return "terraform"
+
+        io.info("Defaulting to copilot mode.")
+        return "copilot"
+
+    @staticmethod
+    def create_strategy(
+        mode: str,
+        clients,
+        ecs_provider: ECS,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        application: Application,
+        addon_name: str,
+        addon_type: str,
+        access: str,
+        env: str,
+        io: ClickIOProvider,
+    ):
+
+        if mode == "terraform":
+            return TerraformConduitStrategy(
+                clients,
+                ecs_provider,
+                application,
+                addon_name,
+                addon_type,
+                access,
+                env,
+                io,
+                vpc_provider=VpcProvider,
+                get_postgres_admin_connection_string=get_postgres_admin_connection_string,
+            )
+        else:
+            return CopilotConduitStrategy(
+                clients,
+                ecs_provider,
+                secrets_provider,
+                cloudformation_provider,
+                application,
+                addon_name,
+                access,
+                env,
+                io,
+                connect_to_addon_client_task=connect_to_addon_client_task,
+                create_addon_client_task=create_addon_client_task,
+            )
+
+
+class Conduit:
+    def __init__(
+        self,
+        application: Application,
+        secrets_provider: Secrets,
+        cloudformation_provider: CloudFormation,
+        ecs_provider: ECS,
+        io: ClickIOProvider = ClickIOProvider(),
+        vpc_provider=VpcProvider,
+        strategy_factory: Optional[ConduitStrategyFactory] = None,
+    ):
+
+        self.application = application
+        self.secrets_provider = secrets_provider
+        self.cloudformation_provider = cloudformation_provider
+        self.ecs_provider = ecs_provider
+        self.io = io
+        self.vpc_provider = vpc_provider
+        self.strategy_factory = strategy_factory or ConduitStrategyFactory()
+
+    def start(self, env: str, addon_name: str, access: str = "read"):
+        self.clients = self._initialise_clients(env)
+        addon_type = self.secrets_provider.get_addon_type(addon_name)
+
+        if (addon_type == "opensearch" or addon_type == "redis") and (access != "read"):
+            access = "read"
+
+        mode = self.strategy_factory.detect_mode(
+            self.clients.get("ecs"),
+            self.application.name,
+            env,
+            addon_name,
+            addon_type,
+            access,
+            self.io,
+        )
+
+        strategy = self.strategy_factory.create_strategy(
+            mode=mode,
+            clients=self.clients,
+            ecs_provider=self.ecs_provider,
+            secrets_provider=self.secrets_provider,
+            cloudformation_provider=self.cloudformation_provider,
+            application=self.application,
+            addon_name=addon_name,
+            addon_type=addon_type,
+            access=access,
+            env=env,
+            io=self.io,
+        )
+
+        data_context = strategy.get_data()
+
+        data_context["task_arns"] = self.ecs_provider.get_ecs_task_arns(
+            data_context["cluster_arn"], data_context["task_def_family"]
+        )
+
+        info_log = (
+            f"Checking if a conduit ECS task is already running for:\n"
+            f"  Addon Name : {addon_name}\n"
+            f"  Addon Type : {addon_type}"
+        )
+
+        if addon_type == "postgres":
+            info_log += f"\n  Access Level : {access}"
+
+        self.io.info(info_log)
+
+        if not data_context["task_arns"]:
+            self.io.info("Creating conduit ECS task...")
+            strategy.start_task(data_context)
+            data_context["task_arns"] = self.ecs_provider.wait_for_task_to_register(
+                data_context["cluster_arn"], data_context["task_def_family"]
+            )
+        else:
+            self.io.info(f"Found a task already running: {data_context['task_arns'][0]}")
+
+        self.io.info(f"Waiting for ECS Exec agent to become available on the conduit task...")
+        self.ecs_provider.ecs_exec_is_available(
+            data_context["cluster_arn"], data_context["task_arns"]
+        )
+
+        self.io.info("Connecting to conduit task...")
+        strategy.exec_task(data_context)
+
+    def _initialise_clients(self, env):
+        return {
+            "ecs": self.application.environments[env].session.client("ecs"),
+            "iam": self.application.environments[env].session.client("iam"),
+            "ssm": self.application.environments[env].session.client("ssm"),
+        }

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/domain/copilot.py

@@ -10,6 +10,7 @@ import botocore.errorfactory
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
+from dbt_platform_helper.domain.plans import PlanLoader
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
@@ -45,6 +46,7 @@ class Copilot:
         kms_provider: KMSProvider,
         session,
         io: ClickIOProvider = ClickIOProvider(),
+        plan_manager: PlanLoader = PlanLoader(),
         yaml_file_provider: YamlFileProvider = YamlFileProvider,
     ):
         self.config_provider = config_provider
@@ -53,6 +55,7 @@ class Copilot:
         self.copilot_templating = copilot_templating
         self.kms_provider = kms_provider
         self.io = io
+        self.plan_manager = plan_manager
         self.yaml_file_provider = yaml_file_provider
         self.session = session
 
@@ -169,7 +172,7 @@ class Copilot:
         def _normalise_keys(source: dict):
             return {k.replace("-", "_"): v for k, v in source.items()}
 
-        addon_plans = self.yaml_file_provider.load(self.PACKAGE_DIR / "addon-plans.yml")
+        addon_plans = self.plan_manager.load()
 
         # load and validate config
         config = self.yaml_file_provider.load(config_file)

dbt_platform_helper-15.2.1/dbt_platform_helper/domain/plans.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+class PlanLoader:
+
+    PROJECT_DIR = Path(__file__).resolve().parent.parent.parent
+
+    def __init__(
+        self,
+        extensions: dict = None,
+        terraform_dir: str = "terraform",
+        loader: YamlFileProvider = YamlFileProvider,
+    ):
+        self.path = terraform_dir
+        self.loader = loader
+        self._cache = {}
+        self.extensions = extensions or {
+            "redis": "elasticache-redis",
+            "opensearch": "opensearch",
+            "postgres": "postgres",
+        }
+
+    def load(self):
+        result = {}
+        for key, value in self.extensions.items():
+            result[key] = self._load_plan(key, f"{self.PROJECT_DIR}/{self.path}/{value}/plans.yml")
+        return result
+
+    def _load_plan(self, name, path):
+        if name in self._cache:
+            return self._cache[name]
+        else:
+            plan = self.loader.load(path)
+            self._cache[name] = plan
+            return plan
+
+    def get_plan_names(self, extension):
+        plans = self.load()
+        return list(plans[extension].keys())

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/entities/platform_config_schema.py

@@ -9,6 +9,10 @@ from schema import Schema
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_SCHEMA_VERSION
+from dbt_platform_helper.domain.plans import PlanLoader
+
+plan_manager = PlanLoader()
+plan_manager.load()
 
 
 class PlatformConfigSchema:
@@ -248,17 +252,7 @@ class PlatformConfigSchema:
     @staticmethod
     def __opensearch_schema() -> dict:
         # TODO: DBTP-1943: Move to OpenSearch provider?
-        _valid_opensearch_plans = Or(
-            "tiny",
-            "small",
-            "small-ha",
-            "medium",
-            "medium-ha",
-            "large",
-            "large-ha",
-            "x-large",
-            "x-large-ha",
-        )
+        _valid_opensearch_plans = Or(*plan_manager.get_plan_names("opensearch"))
 
         return {
             "type": "opensearch",
@@ -285,28 +279,7 @@ class PlatformConfigSchema:
 
     @staticmethod
     def __postgres_schema() -> dict:
-        # TODO: DBTP-1943: Move to Postgres provider?
-        _valid_postgres_plans = Or(
-            "tiny",
-            "small",
-            "small-ha",
-            "small-high-io",
-            "medium",
-            "medium-ha",
-            "medium-high-io",
-            "large",
-            "large-ha",
-            "large-high-io",
-            "x-large",
-            "x-large-ha",
-            "x-large-high-io",
-            "2x-large",
-            "2x-large-ha",
-            "2x-large-high-io",
-            "4x-large",
-            "4x-large-ha",
-            "4x-large-high-io",
-        )
+        _valid_postgres_plans = Or(*plan_manager.get_plan_names("postgres"))
 
         # TODO: DBTP-1943: Move to Postgres provider?
         _valid_postgres_storage_types = Or("gp2", "gp3", "io1", "io2")
@@ -361,21 +334,7 @@ class PlatformConfigSchema:
 
     @staticmethod
     def __redis_schema() -> dict:
-        # TODO: DBTP-1943: move to Redis provider?
-        _valid_redis_plans = Or(
-            "micro",
-            "micro-ha",
-            "tiny",
-            "tiny-ha",
-            "small",
-            "small-ha",
-            "medium",
-            "medium-ha",
-            "large",
-            "large-ha",
-            "x-large",
-            "x-large-ha",
-        )
+        _valid_redis_plans = Or(*plan_manager.get_plan_names("redis"))
 
         return {
             "type": "redis",

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/platform_exception.py

@@ -3,3 +3,7 @@
 # error and abort.
 class PlatformException(Exception):
     pass
+
+
+class ValidationException(PlatformException):
+    pass

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/providers/aws/exceptions.py

@@ -12,16 +12,34 @@ class CreateTaskTimeoutException(AWSException):
         )
 
 
+IMAGE_NOT_FOUND_TEMPLATE = """An image labelled "{image_ref}" could not be found in your image repository. Try the `platform-helper codebase build` command first."""
+
+
 class ImageNotFoundException(AWSException):
     def __init__(self, image_ref: str):
+        super().__init__(IMAGE_NOT_FOUND_TEMPLATE.format(image_ref=image_ref))
+
+
+MULTIPLE_IMAGES_FOUND_TEMPLATE = (
+    'Image reference "{image_ref}" is matched by the following images: {matching_images}'
+)
+
+
+class MultipleImagesFoundException(AWSException):
+    def __init__(self, image_ref: str, matching_images: list[str]):
         super().__init__(
-            f"""An image labelled "{image_ref}" could not be found in your image repository. Try the `platform-helper codebase build` command first."""
+            MULTIPLE_IMAGES_FOUND_TEMPLATE.format(
+                image_ref=image_ref, matching_images=", ".join(sorted(matching_images))
+            )
         )
 
 
+REPOSITORY_NOT_FOUND_TEMPLATE = """The ECR repository "{repository}" could not be found."""
+
+
 class RepositoryNotFoundException(AWSException):
     def __init__(self, repository: str):
-        super().__init__(f"""The ECR repository "{repository}" could not be found.""")
+        super().__init__(REPOSITORY_NOT_FOUND_TEMPLATE.format(repository=repository))
 
 
 class LogGroupNotFoundException(AWSException):

{dbt_platform_helper-15.1.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/providers/copilot.py

@@ -1,4 +1,5 @@
 import json
+import subprocess
 import time
 
 from botocore.exceptions import ClientError
@@ -13,7 +14,6 @@ from dbt_platform_helper.utils.messages import abort_with_error
 def create_addon_client_task(
     iam_client,
     ssm_client,
-    subprocess,
     application: Application,
     env: str,
     addon_type: str,
@@ -31,7 +31,6 @@ def create_addon_client_task(
         elif access == "admin":
             create_postgres_admin_task(
                 ssm_client,
-                subprocess,
                 application,
                 addon_name,
                 addon_type,
@@ -71,15 +70,8 @@ def create_addon_client_task(
     )
 
 
-def create_postgres_admin_task(
-    ssm_client,
-    subprocess,
-    app: Application,
-    addon_name: str,
-    addon_type: str,
-    env: str,
-    secret_name: str,
-    task_name: str,
+def get_postgres_admin_connection_string(
+    ssm_client, secret_name: str, app: Application, env: str, addon_name: str
 ):
     read_only_secret_name = secret_name + "_READ_ONLY_USER"
     master_secret_name = (
@@ -94,6 +86,23 @@ def create_postgres_admin_task(
         )
     )
 
+    return connection_string
+
+
+def create_postgres_admin_task(
+    ssm_client,
+    app: Application,
+    addon_name: str,
+    addon_type: str,
+    env: str,
+    secret_name: str,
+    task_name: str,
+):
+
+    connection_string = get_postgres_admin_connection_string(
+        ssm_client, secret_name, app, env, addon_name
+    )
+
     subprocess.call(
         f"copilot task run --app {app.name} --env {env} "
         f"--task-group-name {task_name} "
@@ -121,7 +130,6 @@ def _temp_until_refactor_get_ecs_task_arns(ecs_client, cluster_arn: str, task_na
 
 def connect_to_addon_client_task(
     ecs_client,
-    subprocess,
     application_name,
     env,
     cluster_arn,