dbt-platform-helper 12.5.1__py3-none-any.whl → 12.6.0__py3-none-any.whl

dbt_platform_helper/domain/copilot_environment.py

@@ -1,162 +1,168 @@
 from collections import defaultdict
 from pathlib import Path
+from typing import Callable
 
-import boto3
 import click
+from boto3 import Session
 
-from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.domain.terraform_environment import (
+    EnvironmentNotFoundException,
+)
+from dbt_platform_helper.providers.cloudformation import CloudFormation
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
-from dbt_platform_helper.providers.load_balancers import find_https_listener
-from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.providers.load_balancers import (
+    get_https_certificate_for_application,
+)
+from dbt_platform_helper.providers.vpc import Vpc
+from dbt_platform_helper.providers.vpc import VpcNotFoundForNameException
+from dbt_platform_helper.providers.vpc import VpcProvider
 from dbt_platform_helper.utils.template import S3_CROSS_ACCOUNT_POLICY
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
 
 
-# TODO - move helper functions into suitable provider classes
-def get_subnet_ids(session, vpc_id, environment_name):
-    subnets = session.client("ec2").describe_subnets(
-        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
-    )["Subnets"]
-
-    if not subnets:
-        click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
-        raise click.Abort
-
-    public_tag = {"Key": "subnet_type", "Value": "public"}
-    public_subnets = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
-    private_tag = {"Key": "subnet_type", "Value": "private"}
-    private_subnets = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
-
-    # This call and the method declaration can be removed when we stop using AWS Copilot to deploy the services
-    public_subnets, private_subnets = _match_subnet_id_order_to_cloudformation_exports(
-        session,
-        environment_name,
-        public_subnets,
-        private_subnets,
-    )
-
-    return public_subnets, private_subnets
-
-
-def _match_subnet_id_order_to_cloudformation_exports(
-    session, environment_name, public_subnets, private_subnets
-):
-    public_subnet_exports = []
-    private_subnet_exports = []
-    for page in session.client("cloudformation").get_paginator("list_exports").paginate():
-        for export in page["Exports"]:
-            if f"-{environment_name}-" in export["Name"]:
-                if export["Name"].endswith("-PublicSubnets"):
-                    public_subnet_exports = export["Value"].split(",")
-                if export["Name"].endswith("-PrivateSubnets"):
-                    private_subnet_exports = export["Value"].split(",")
-
-    # If the elements match, regardless of order, use the list from the CloudFormation exports
-    if set(public_subnets) == set(public_subnet_exports):
-        public_subnets = public_subnet_exports
-    if set(private_subnets) == set(private_subnet_exports):
-        private_subnets = private_subnet_exports
-
-    return public_subnets, private_subnets
-
-
-def get_cert_arn(session, application, env_name):
-    try:
-        arn = find_https_certificate(session, application, env_name)
-    except:
-        click.secho(
-            f"No certificate found with domain name matching environment {env_name}.", fg="red"
+class CopilotEnvironment:
+    def __init__(
+        self,
+        config_provider: ConfigProvider,
+        vpc_provider: VpcProvider = None,
+        cloudformation_provider: CloudFormation = None,
+        session: Session = None,  # TODO - this is a temporary fix, will fall away once the Loadbalancer provider is in place.
+        copilot_templating=None,
+        echo: Callable[[str], str] = click.secho,
+    ):
+        self.config_provider = config_provider
+        self.vpc_provider = vpc_provider
+        self.copilot_templating = copilot_templating or CopilotTemplating(
+            file_provider=FileProvider(),
         )
-        raise click.Abort
+        self.echo = echo
+        self.session = session
+        self.cloudformation_provider = cloudformation_provider
 
-    return arn
+    def generate(self, environment_name: str) -> None:
 
+        platform_config = self.config_provider.get_enriched_config()
 
-def get_vpc_id(session, env_name, vpc_name=None):
-    if not vpc_name:
-        vpc_name = f"{session.profile_name}-{env_name}"
+        if environment_name not in platform_config.get("environments").keys():
+            raise EnvironmentNotFoundException(
+                f"Error: cannot generate copilot manifests for environment {environment_name}.  It does not exist in your configuration"
+            )
 
-    filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
-    vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+        env_config = platform_config["environments"][environment_name]
+        profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
 
-    if not vpcs:
-        filters[0]["Values"] = [session.profile_name]
-        vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+        self.echo(f"Using {profile_for_environment} for this AWS session")
 
-    if not vpcs:
-        click.secho(
-            f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
-        )
-        raise click.Abort
-
-    return vpcs[0]["VpcId"]
-
-
-def _generate_copilot_environment_manifests(
-    environment_name, application_name, env_config, session
-):
-    env_template = setup_templates().get_template("env/manifest.yml")
-    vpc_name = env_config.get("vpc", None)
-    vpc_id = get_vpc_id(session, environment_name, vpc_name)
-    pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id, environment_name)
-    cert_arn = get_cert_arn(session, application_name, environment_name)
-    contents = env_template.render(
-        {
-            "name": environment_name,
-            "vpc_id": vpc_id,
-            "pub_subnet_ids": pub_subnet_ids,
-            "priv_subnet_ids": priv_subnet_ids,
-            "certificate_arn": cert_arn,
-        }
-    )
-    click.echo(
-        FileProvider.mkfile(
-            ".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
+        app_name = platform_config["application"]
+
+        certificate_arn = get_https_certificate_for_application(
+            self.session, app_name, environment_name
         )
-    )
 
+        vpc = self._get_environment_vpc(
+            self.session, app_name, environment_name, env_config.get("vpc", None)
+        )
 
-def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
-    listener_arn = find_https_listener(session, app, env)
-    cert_client = session.client("elbv2")
-    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
-        "Certificates"
-    ]
+        copilot_environment_manifest = self.copilot_templating.generate_copilot_environment_manifest(
+            environment_name=environment_name,
+            # We need to correct the subnet id order before adding it to the template. See pydoc on below method for details.
+            vpc=self._match_subnet_id_order_to_cloudformation_exports(environment_name, vpc),
+            cert_arn=certificate_arn,
+        )
 
-    try:
-        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
-    except StopIteration:
-        raise CertificateNotFoundException()
+        self.echo(
+            self.copilot_templating.write_environment_manifest(
+                environment_name, copilot_environment_manifest
+            )
+        )
 
-    return certificate_arn
+    # TODO: There should always be a vpc_name as defaults have been applied to the config.  This function can
+    # probably fall away. We shouldn't need to check 3 different names (vpc_name, session.profile_name, {session.profile_name}-{env_name})
+    # To be checked.
+    def _get_environment_vpc(self, session: Session, app_name, env_name: str, vpc_name: str) -> Vpc:
 
+        if not vpc_name:
+            vpc_name = f"{session.profile_name}-{env_name}"
 
-class CertificateNotFoundException(PlatformException):
-    pass
+        try:
+            vpc = self.vpc_provider.get_vpc(app_name, env_name, vpc_name)
+        except VpcNotFoundForNameException:
+            vpc = self.vpc_provider.get_vpc(app_name, env_name, session.profile_name)
 
+        if not vpc:
+            raise VpcNotFoundForNameException
 
-class CopilotEnvironment:
-    def __init__(self, config_provider):
-        self.config_provider = config_provider
+        return vpc
 
-    def generate(self, environment_name):
-        config = self.config_provider.load_and_validate_platform_config()
-        enriched_config = self.config_provider.apply_environment_defaults(config)
+    def _match_subnet_id_order_to_cloudformation_exports(
+        self, environment_name: str, vpc: Vpc
+    ) -> Vpc:
+        """
+        Addresses an issue identified in DBTP-1524 'If the order of the subnets
+        in the environment manifest has changed, copilot env deploy tries to do
+        destructive changes.'.
 
-        env_config = enriched_config["environments"][environment_name]
-        profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
-        click.secho(f"Using {profile_for_environment} for this AWS session")
-        session = get_aws_session_or_abort(profile_for_environment)
+        Takes a Vpc object which has a private and public subnets attribute and
+        sorts them to match the order within cfn exports.
+        """
 
-        _generate_copilot_environment_manifests(
-            environment_name, enriched_config["application"], env_config, session
+        exports = self.cloudformation_provider.get_cloudformation_exports_for_environment(
+            environment_name
         )
 
+        public_subnet_exports = []
+        private_subnet_exports = []
+
+        for export in exports:
+            if export["Name"].endswith("-PublicSubnets"):
+                public_subnet_exports = export["Value"].split(",")
+            elif export["Name"].endswith("-PrivateSubnets"):
+                private_subnet_exports = export["Value"].split(",")
+
+        # If the elements match, regardless of order, use the list from the CloudFormation exports
+        if set(vpc.public_subnets) == set(public_subnet_exports):
+            vpc.public_subnets = public_subnet_exports
+        if set(vpc.private_subnets) == set(private_subnet_exports):
+            vpc.private_subnets = private_subnet_exports
+
+        return vpc
+
 
 class CopilotTemplating:
-    def __init__(self, mkfile_fn=FileProvider.mkfile):
-        self.mkfile_fn = mkfile_fn
+    def __init__(
+        self,
+        file_provider: FileProvider = None,
+        # TODO file_provider can be moved up a layer.  File writing can be the responsibility of CopilotEnvironment generate
+        # Or we align with PlatformTerraformManifestGenerator and rename from Templating to reflect the file writing responsibility
+    ):
+        self.file_provider = file_provider
+        self.templates = setup_templates()
+
+    def generate_copilot_environment_manifest(
+        self, environment_name: str, vpc: Vpc, cert_arn: str
+    ) -> str:
+        env_template = self.templates.get_template("env/manifest.yml")
+
+        return env_template.render(
+            {
+                "name": environment_name,
+                "vpc_id": vpc.id,
+                "pub_subnet_ids": vpc.public_subnets,
+                "priv_subnet_ids": vpc.private_subnets,
+                "certificate_arn": cert_arn,
+            }
+        )
+
+    def write_environment_manifest(self, environment_name: str, manifest_contents: str) -> str:
+
+        return self.file_provider.mkfile(
+            ".",
+            f"copilot/environments/{environment_name}/manifest.yml",
+            manifest_contents,
+            overwrite=True,
+        )
 
     def generate_cross_account_s3_policies(self, environments: dict, extensions):
         resource_blocks = defaultdict(list)
@@ -190,15 +196,13 @@ class CopilotTemplating:
             click.echo("\n>>> No cross-environment S3 policies to create.\n")
             return
 
-        templates = setup_templates()
-
         for service in sorted(resource_blocks.keys()):
             resources = resource_blocks[service]
             click.echo(f"\n>>> Creating S3 cross account policies for {service}.\n")
-            template = templates.get_template(S3_CROSS_ACCOUNT_POLICY)
+            template = self.templates.get_template(S3_CROSS_ACCOUNT_POLICY)
             file_content = template.render({"resources": resources})
             output_dir = Path(".").absolute()
             file_path = f"copilot/{service}/addons/s3-cross-account-policy.yml"
 
-            self.mkfile_fn(output_dir, file_path, file_content, True)
+            self.file_provider.mkfile(output_dir, file_path, file_content, True)
             click.echo(f"File {file_path} created")

dbt_platform_helper/domain/database_copy.py

@@ -3,22 +3,22 @@ from collections.abc import Callable
 from pathlib import Path
 
 import boto3
-import click
 from boto3 import Session
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
 from dbt_platform_helper.domain.maintenance_page import MaintenancePage
-from dbt_platform_helper.providers.aws import AWSException
 from dbt_platform_helper.providers.config import ConfigProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.io import ClickIOProviderException
 from dbt_platform_helper.providers.vpc import Vpc
 from dbt_platform_helper.providers.vpc import VpcProvider
+from dbt_platform_helper.providers.vpc import VpcProviderException
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import ApplicationNotFoundException
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import get_connection_string
 from dbt_platform_helper.utils.aws import wait_for_log_group_to_exist
-from dbt_platform_helper.utils.messages import abort_with_error
 
 
 class DatabaseCopy:
@@ -33,12 +33,8 @@ class DatabaseCopy:
         db_connection_string: Callable[
             [Session, str, str, str, Callable], str
         ] = get_connection_string,
-        maintenance_page_provider: Callable[
-            [str, str, list[str], str, str], None
-        ] = MaintenancePage(),
-        input: Callable[[str], str] = click.prompt,
-        echo: Callable[[str], str] = click.secho,
-        abort: Callable[[str], None] = abort_with_error,
+        maintenance_page: Callable[[str, str, list[str], str, str], None] = MaintenancePage,
+        io: ClickIOProvider = ClickIOProvider(),
         config_provider: ConfigProvider = ConfigProvider(ConfigValidator()),
     ):
         self.app = app
@@ -46,15 +42,14 @@ class DatabaseCopy:
         self.auto_approve = auto_approve
         self.vpc_provider = vpc_provider
         self.db_connection_string = db_connection_string
-        self.maintenance_page_provider = maintenance_page_provider
-        self.input = input
-        self.echo = echo
-        self.abort = abort
+        self.io = io
         self.config_provider = config_provider
 
         if not self.app:
             if not Path(PLATFORM_CONFIG_FILE).exists():
-                self.abort("You must either be in a deploy repo, or provide the --app option.")
+                self.io.abort_with_error(
+                    "You must either be in a deploy repo, or provide the --app option."
+                )
 
             config = self.config_provider.load_and_validate_platform_config()
             self.app = config["application"]
@@ -62,7 +57,9 @@ class DatabaseCopy:
         try:
             self.application = load_application(self.app)
         except ApplicationNotFoundException:
-            abort(f"No such application '{app}'.")
+            self.io.abort_with_error(f"No such application '{app}'.")
+
+        self.maintenance_page = maintenance_page(self.application)
 
     def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, filename: str):
         vpc_name = self.enrich_vpc_name(env, vpc_name)
@@ -70,7 +67,7 @@ class DatabaseCopy:
         environments = self.application.environments
         environment = environments.get(env)
         if not environment:
-            self.abort(
+            self.io.abort_with_error(
                 f"No such environment '{env}'. Available environments are: {', '.join(environments.keys())}"
             )
 
@@ -78,9 +75,9 @@ class DatabaseCopy:
 
         try:
             vpc_provider = self.vpc_provider(env_session)
-            vpc_config = vpc_provider.get_vpc_info_by_name(self.app, env, vpc_name)
-        except AWSException as ex:
-            self.abort(str(ex))
+            vpc_config = vpc_provider.get_vpc(self.app, env, vpc_name)
+        except VpcProviderException as ex:
+            self.io.abort_with_error(str(ex))
 
         database_identifier = f"{self.app}-{env}-{self.database}"
 
@@ -89,31 +86,32 @@ class DatabaseCopy:
                 env_session, self.app, env, database_identifier
             )
         except Exception as exc:
-            self.abort(f"{exc} (Database: {database_identifier})")
+            self.io.abort_with_error(f"{exc} (Database: {database_identifier})")
 
         try:
             task_arn = self.run_database_copy_task(
                 env_session, env, vpc_config, is_dump, db_connection_string, filename
             )
         except Exception as exc:
-            self.abort(f"{exc} (Account id: {self.account_id(env)})")
+            self.io.abort_with_error(f"{exc} (Account id: {self.account_id(env)})")
 
         if is_dump:
             message = f"Dumping {self.database} from the {env} environment into S3"
         else:
             message = f"Loading data into {self.database} in the {env} environment from S3"
 
-        self.echo(message, fg="white", bold=True)
-        self.echo(
-            f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
-            fg="white",
+        self.io.info(message)
+        self.io.info(
+            f"Task {task_arn} started. Waiting for it to complete (this may take some time)..."
         )
         self.tail_logs(is_dump, env)
 
     def enrich_vpc_name(self, env, vpc_name):
         if not vpc_name:
             if not Path(PLATFORM_CONFIG_FILE).exists():
-                self.abort("You must either be in a deploy repo, or provide the vpc name option.")
+                self.io.abort_with_error(
+                    "You must either be in a deploy repo, or provide the vpc name option."
+                )
             config = self.config_provider.load_and_validate_platform_config()
             env_config = self.config_provider.apply_environment_defaults(config)["environments"]
             vpc_name = env_config.get(env, {}).get("vpc")
@@ -147,7 +145,7 @@ class DatabaseCopy:
             ],
             networkConfiguration={
                 "awsvpcConfiguration": {
-                    "subnets": vpc_config.subnets,
+                    "subnets": vpc_config.private_subnets,
                     "securityGroups": vpc_config.security_groups,
                     "assignPublicIp": "DISABLED",
                 }
@@ -183,26 +181,27 @@ class DatabaseCopy:
     ):
         to_vpc = self.enrich_vpc_name(to_env, to_vpc)
         if not no_maintenance_page:
-            self.maintenance_page_provider.activate(self.app, to_env, services, template, to_vpc)
+            self.maintenance_page.activate(to_env, services, template, to_vpc)
         self.dump(from_env, from_vpc, f"data_dump_{to_env}")
         self.load(to_env, to_vpc, f"data_dump_{to_env}")
         if not no_maintenance_page:
-            self.maintenance_page_provider.deactivate(self.app, to_env)
+            self.maintenance_page.deactivate(to_env)
 
     def is_confirmed_ready_to_load(self, env: str) -> bool:
         if self.auto_approve:
             return True
-
-        user_input = self.input(
-            f"\nWARNING: the load operation is destructive and will delete the {self.database} database in the {env} environment. Continue? (y/n)"
-        )
-        return user_input.lower().strip() in ["y", "yes"]
+        try:
+            return self.io.confirm(
+                f"\nWARNING: the load operation is destructive and will delete the {self.database} database in the {env} environment. Continue?"
+            )
+        except ClickIOProviderException:
+            return False
 
     def tail_logs(self, is_dump: bool, env: str):
         action = "dump" if is_dump else "load"
         log_group_name = f"/ecs/{self.app}-{env}-{self.database}-{action}"
         log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id(env)}:log-group:{log_group_name}"
-        self.echo(f"Tailing {log_group_name} logs", fg="yellow")
+        self.io.warn(f"Tailing {log_group_name} logs")
         session = self.application.environments[env].session
         log_client = session.client("logs")
         wait_for_log_group_to_exist(log_client, log_group_name)
@@ -220,9 +219,11 @@ class DatabaseCopy:
                     match = re.match(r"(Stopping|Aborting) data (load|dump).*", message)
                     if match:
                         if match.group(1) == "Aborting":
-                            self.abort("Task aborted abnormally. See logs above for details.")
+                            self.io.abort_with_error(
+                                "Task aborted abnormally. See logs above for details."
+                            )
                         stopped = True
-                    self.echo(message)
+                    self.io.info(message)
 
     def account_id(self, env):
         envs = self.application.environments