dbt-platform-helper 12.5.1__py3-none-any.whl → 12.6.0__py3-none-any.whl

dbt_platform_helper/domain/maintenance_page.py

@@ -11,14 +11,16 @@ import boto3
 import click
 
 from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
 from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
 from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
-from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.providers.load_balancers import (
+    get_https_listener_for_application,
+)
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import Environment
 from dbt_platform_helper.utils.application import Service
-from dbt_platform_helper.utils.application import load_application
 
 
 class MaintenancePageException(PlatformException):
@@ -26,134 +28,272 @@ class MaintenancePageException(PlatformException):
 
 
 class LoadBalancedWebServiceNotFoundException(MaintenancePageException):
-    pass
+    def __init__(self, application_name):
+        super().__init__(f"No services deployed yet to {application_name} ")
+
+
+def get_maintenance_page_type(session: boto3.Session, listener_arn: str) -> Union[str, None]:
+    lb_client = session.client("elbv2")
+
+    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
+    tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
+
+    maintenance_page_type = None
+    for description in tag_descriptions:
+        tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+        if tags.get("name") == "MaintenancePage":
+            maintenance_page_type = tags.get("type")
+
+    return maintenance_page_type
+
+
+def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
+    account_name = f"{application_environment.session.profile_name}-vpc"
+    vpc_name = vpc if vpc else account_name
+    ssm_client = application_environment.session.client("ssm")
+
+    try:
+        param_value = ssm_client.get_parameter(Name=f"/{vpc_name}/EGRESS_IPS")["Parameter"]["Value"]
+    except ssm_client.exceptions.ParameterNotFound:
+        click.secho(f"No parameter found with name: /{vpc_name}/EGRESS_IPS")
+        raise click.Abort
+
+    return [ip.strip() for ip in param_value.split(",")]
+
+
+def add_maintenance_page(
+    session: boto3.Session,
+    listener_arn: str,
+    app: str,
+    env: str,
+    services: List[Service],
+    allowed_ips: tuple,
+    template: str = "default",
+):
+    lb_client = session.client("elbv2")
+    maintenance_page_content = get_maintenance_page_template(template)
+    bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
+
+    rule_priority = itertools.count(start=1)
+
+    for svc in services:
+        target_group_arn = find_target_group(app, env, svc.name, session)
+
+        # not all of an application's services are guaranteed to have been deployed to an environment
+        if not target_group_arn:
+            continue
+
+        for ip in allowed_ips:
+            create_header_rule(
+                lb_client,
+                listener_arn,
+                target_group_arn,
+                "X-Forwarded-For",
+                [ip],
+                "AllowedIps",
+                next(rule_priority),
+            )
+            create_source_ip_rule(
+                lb_client,
+                listener_arn,
+                target_group_arn,
+                [ip],
+                "AllowedSourceIps",
+                next(rule_priority),
+            )
+
+        create_header_rule(
+            lb_client,
+            listener_arn,
+            target_group_arn,
+            "Bypass-Key",
+            [bypass_value],
+            "BypassIpFilter",
+            next(rule_priority),
+        )
+
+        click.secho(
+            f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
+        )
+
+    lb_client.create_rule(
+        ListenerArn=listener_arn,
+        Priority=next(rule_priority),
+        Conditions=[
+            {
+                "Field": "path-pattern",
+                "PathPatternConfig": {"Values": ["/*"]},
+            }
+        ],
+        Actions=[
+            {
+                "Type": "fixed-response",
+                "FixedResponseConfig": {
+                    "StatusCode": "503",
+                    "ContentType": "text/html",
+                    "MessageBody": maintenance_page_content,
+                },
+            }
+        ],
+        Tags=[
+            {"Key": "name", "Value": "MaintenancePage"},
+            {"Key": "type", "Value": template},
+        ],
+    )
+
+
+def remove_maintenance_page(session: boto3.Session, listener_arn: str):
+    lb_client = session.client("elbv2")
+
+    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
+    tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
+
+    for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
+        deleted = delete_listener_rule(tag_descriptions, name, lb_client)
+
+        if name == "MaintenancePage" and not deleted:
+            raise ListenerRuleNotFoundException()
 
 
 class MaintenancePage:
     def __init__(
         self,
-        user_prompt_callback: Callable[[str], bool] = click.confirm,
-        echo: Callable[[str], str] = click.secho,
+        application: Application,
+        io: ClickIOProvider = ClickIOProvider(),
+        get_https_listener_for_application: Callable[
+            [boto3.Session, str, str], str
+        ] = get_https_listener_for_application,
+        # TODO refactor get_maintenance_page_type, add_maintenance_page, remove_maintenance_page into MaintenancePage class with LoadBalancerProvider as the dependency
+        get_maintenance_page_type: Callable[
+            [boto3.Session, str], Union[str, None]
+        ] = get_maintenance_page_type,
+        get_env_ips: Callable[[str, Environment], List[str]] = get_env_ips,
+        add_maintenance_page: Callable[
+            [boto3.Session, str, str, str, List[Service], tuple, str], None
+        ] = add_maintenance_page,
+        remove_maintenance_page: Callable[[boto3.Session, str], None] = remove_maintenance_page,
     ):
-        self.user_prompt_callback = user_prompt_callback
-        self.echo = echo
-
-    def _get_deployed_load_balanced_web_services(self, app: Application, svc: str):
+        self.application = application
+        self.get_https_listener_for_application = get_https_listener_for_application
+        self.io = io
+        self.get_maintenance_page_type = get_maintenance_page_type
+        self.get_env_ips = get_env_ips
+        self.add_maintenance_page = add_maintenance_page
+        self.remove_maintenance_page = remove_maintenance_page
+
+    def _get_deployed_load_balanced_web_services(self, app: Application, svc: List[str]):
         if "*" in svc:
             services = [s for s in app.services.values() if s.kind == "Load Balanced Web Service"]
         else:
-            all_services = [get_app_service(app.name, s) for s in list(svc)]
+            all_services = [get_app_service(app, s) for s in list(svc)]
             services = [s for s in all_services if s.kind == "Load Balanced Web Service"]
         if not services:
-            raise LoadBalancedWebServiceNotFoundException
+            raise LoadBalancedWebServiceNotFoundException(app.name)
         return services
 
-    def activate(self, app, env, svc, template, vpc):
-        try:
-            services = self._get_deployed_load_balanced_web_services(load_application(app), svc)
-        except LoadBalancedWebServiceNotFoundException:
-            # TODO DBTP-1643 - this bit of logic does not depend on env, so env shouldn't really be in the exception
-            # message
-            # Exception should be propagated to command and caught there.
-            self.echo(f"No services deployed yet to {app} environment {env}", fg="red")
-            raise click.Abort
-
-        application_environment = get_app_environment(app, env)
+    def activate(self, env: str, services: List[str], template: str, vpc: Union[str, None]):
+
+        services = self._get_deployed_load_balanced_web_services(self.application, services)
+        application_environment = get_app_environment(self.application, env)
+
         try:
-            https_listener = find_https_listener(application_environment.session, app, env)
-            current_maintenance_page = get_maintenance_page(
+            https_listener = self.get_https_listener_for_application(
+                application_environment.session, self.application.name, env
+            )
+            current_maintenance_page = self.get_maintenance_page_type(
                 application_environment.session, https_listener
             )
             remove_current_maintenance_page = False
             if current_maintenance_page:
-                remove_current_maintenance_page = self.user_prompt_callback(
+                remove_current_maintenance_page = self.io.confirm(
                     f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
-                    f"environment in {app}.\nWould you like to replace it with a '{template}' "
+                    f"environment in {self.application.name}.\nWould you like to replace it with a '{template}' "
                     f"maintenance page?"
                 )
                 if not remove_current_maintenance_page:
-                    raise click.Abort
+                    return
 
-            if remove_current_maintenance_page or self.user_prompt_callback(
+            if remove_current_maintenance_page or self.io.confirm(
                 f"You are about to enable the '{template}' maintenance page for the {env} "
-                f"environment in {app}.\nWould you like to continue?"
+                f"environment in {self.application.name}.\nWould you like to continue?"
             ):
                 if current_maintenance_page and remove_current_maintenance_page:
-                    remove_maintenance_page(application_environment.session, https_listener)
+                    self.remove_maintenance_page(application_environment.session, https_listener)
 
-                allowed_ips = get_env_ips(vpc, application_environment)
+                allowed_ips = self.get_env_ips(vpc, application_environment)
 
-                add_maintenance_page(
+                self.add_maintenance_page(
                     application_environment.session,
                     https_listener,
-                    app,
+                    self.application.name,
                     env,
                     services,
                     allowed_ips,
                     template,
                 )
-                self.echo(
-                    f"Maintenance page '{template}' added for environment {env} in application {app}",
-                    fg="green",
+                self.io.info(
+                    f"Maintenance page '{template}' added for environment {env} in application {self.application.name}",
                 )
-            else:
-                raise click.Abort
 
         except LoadBalancerNotFoundException:
-            self.echo(
-                f"No load balancer found for environment {env} in the application {app}.", fg="red"
+            # TODO push exception to command layer
+            self.io.abort_with_error(
+                f"No load balancer found for environment {env} in the application {self.application.name}.",
             )
-            raise click.Abort
 
         except ListenerNotFoundException:
-            self.echo(
-                f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
+            # TODO push exception to command layer
+            self.io.abort_with_error(
+                f"No HTTPS listener found for environment {env} in the application {self.application.name}.",
             )
-            raise click.Abort
 
-    def deactivate(self, app, env):
-        application_environment = get_app_environment(app, env)
+    def deactivate(self, env: str):
+        application_environment = get_app_environment(self.application, env)
 
         try:
-            https_listener = find_https_listener(application_environment.session, app, env)
-            current_maintenance_page = get_maintenance_page(
+            https_listener = self.get_https_listener_for_application(
+                application_environment.session, self.application.name, env
+            )
+            current_maintenance_page = self.get_maintenance_page_type(
                 application_environment.session, https_listener
             )
+
+            # TODO discuss, reduce number of return statements but more nested if statements
             if not current_maintenance_page:
-                self.echo("There is no current maintenance page to remove", fg="red")
-                raise click.Abort
+                self.io.warn("There is no current maintenance page to remove")
+                return
 
-            if not self.user_prompt_callback(
+            if not self.io.confirm(
                 f"There is currently a '{current_maintenance_page}' maintenance page, "
                 f"would you like to remove it?"
             ):
-                raise click.Abort
+                return
 
-            remove_maintenance_page(application_environment.session, https_listener)
-            self.echo(
-                f"Maintenance page removed from environment {env} in application {app}", fg="green"
+            self.remove_maintenance_page(application_environment.session, https_listener)
+            self.io.info(
+                f"Maintenance page removed from environment {env} in application {self.application.name}",
             )
 
         except LoadBalancerNotFoundException:
-            self.echo(
-                f"No load balancer found for environment {env} in the application {app}.", fg="red"
+            # TODO push exception to command layer
+            self.io.abort_with_error(
+                f"No load balancer found for environment {env} in the application {self.application.name}.",
             )
-            raise click.Abort
 
         except ListenerNotFoundException:
-            self.echo(
-                f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
+            # TODO push exception to command layer
+            self.io.abort_with_error(
+                f"No HTTPS listener found for environment {env} in the application {self.application.name}.",
             )
-            raise click.Abort
 
 
-def get_app_service(app_name: str, svc_name: str) -> Service:
-    application = load_application(app_name)
+def get_app_service(application: Application, svc_name: str) -> Service:
     application_service = application.services.get(svc_name)
 
     if not application_service:
+        # TODO raise exception instead of abort
         click.secho(
-            f"The service {svc_name} was not found in the application {app_name}. "
+            f"The service {svc_name} was not found in the application {application.name}. "
             f"It either does not exist, or has not been deployed.",
             fg="red",
         )
@@ -162,13 +302,12 @@ def get_app_service(app_name: str, svc_name: str) -> Service:
     return application_service
 
 
-def get_app_environment(app_name: str, env_name: str) -> Environment:
-    application = load_application(app_name)
+def get_app_environment(application: Application, env_name: str) -> Environment:
     application_environment = application.environments.get(env_name)
 
     if not application_environment:
         click.secho(
-            f"The environment {env_name} was not found in the application {app_name}. "
+            f"The environment {env_name} was not found in the application {application.name}. "
             f"It either does not exist, or has not been deployed.",
             fg="red",
         )
@@ -177,36 +316,6 @@ def get_app_environment(app_name: str, env_name: str) -> Environment:
     return application_environment
 
 
-def get_maintenance_page(session: boto3.Session, listener_arn: str) -> Union[str, None]:
-    lb_client = session.client("elbv2")
-
-    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
-    tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
-
-    maintenance_page_type = None
-    for description in tag_descriptions:
-        tags = {t["Key"]: t["Value"] for t in description["Tags"]}
-        if tags.get("name") == "MaintenancePage":
-            maintenance_page_type = tags.get("type")
-
-    return maintenance_page_type
-
-
-def remove_maintenance_page(session: boto3.Session, listener_arn: str):
-    lb_client = session.client("elbv2")
-
-    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
-    tag_descriptions = lb_client.describe_tags(ResourceArns=[r["RuleArn"] for r in rules])[
-        "TagDescriptions"
-    ]
-
-    for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
-        deleted = delete_listener_rule(tag_descriptions, name, lb_client)
-
-        if name == "MaintenancePage" and not deleted:
-            raise ListenerRuleNotFoundException()
-
-
 def get_rules_tag_descriptions(rules: list, lb_client):
     tag_descriptions = []
     chunk_size = 20
@@ -233,88 +342,6 @@ def delete_listener_rule(tag_descriptions: list, tag_name: str, lb_client: boto3
     return current_rule_arn
 
 
-def add_maintenance_page(
-    session: boto3.Session,
-    listener_arn: str,
-    app: str,
-    env: str,
-    services: List[Service],
-    allowed_ips: tuple,
-    template: str = "default",
-):
-    lb_client = session.client("elbv2")
-    maintenance_page_content = get_maintenance_page_template(template)
-    bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
-
-    rule_priority = itertools.count(start=1)
-
-    for svc in services:
-        target_group_arn = find_target_group(app, env, svc.name, session)
-
-        # not all of an application's services are guaranteed to have been deployed to an environment
-        if not target_group_arn:
-            continue
-
-        for ip in allowed_ips:
-            create_header_rule(
-                lb_client,
-                listener_arn,
-                target_group_arn,
-                "X-Forwarded-For",
-                [ip],
-                "AllowedIps",
-                next(rule_priority),
-            )
-            create_source_ip_rule(
-                lb_client,
-                listener_arn,
-                target_group_arn,
-                [ip],
-                "AllowedSourceIps",
-                next(rule_priority),
-            )
-
-        create_header_rule(
-            lb_client,
-            listener_arn,
-            target_group_arn,
-            "Bypass-Key",
-            [bypass_value],
-            "BypassIpFilter",
-            next(rule_priority),
-        )
-
-        click.secho(
-            f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
-            fg="green",
-        )
-
-    lb_client.create_rule(
-        ListenerArn=listener_arn,
-        Priority=next(rule_priority),
-        Conditions=[
-            {
-                "Field": "path-pattern",
-                "PathPatternConfig": {"Values": ["/*"]},
-            }
-        ],
-        Actions=[
-            {
-                "Type": "fixed-response",
-                "FixedResponseConfig": {
-                    "StatusCode": "503",
-                    "ContentType": "text/html",
-                    "MessageBody": maintenance_page_content,
-                },
-            }
-        ],
-        Tags=[
-            {"Key": "name", "Value": "MaintenancePage"},
-            {"Key": "type", "Value": template},
-        ],
-    )
-
-
 def get_maintenance_page_template(template) -> str:
     template_contents = (
         Path(__file__)
@@ -472,17 +499,3 @@ def get_host_conditions(lb_client: boto3.client, listener_arn: str, target_group
     ]
 
     return conditions
-
-
-def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
-    account_name = f"{application_environment.session.profile_name}-vpc"
-    vpc_name = vpc if vpc else account_name
-    ssm_client = application_environment.session.client("ssm")
-
-    try:
-        param_value = ssm_client.get_parameter(Name=f"/{vpc_name}/EGRESS_IPS")["Parameter"]["Value"]
-    except ssm_client.exceptions.ParameterNotFound:
-        click.secho(f"No parameter found with name: /{vpc_name}/EGRESS_IPS")
-        raise click.Abort
-
-    return [ip.strip() for ip in param_value.split(",")]

dbt_platform_helper/domain/pipelines.py

@@ -8,6 +8,7 @@ from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENTS_KEY
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.aws import get_account_details
 from dbt_platform_helper.utils.aws import get_public_repository_arn
@@ -22,16 +23,14 @@ class Pipelines:
     def __init__(
         self,
         config_provider: ConfigProvider,
-        echo: Callable[[str], str],
-        abort: Callable[[str], None],
         get_git_remote: Callable[[], str],
         get_codestar_arn: Callable[[str], str],
+        io: ClickIOProvider = ClickIOProvider(),
     ):
         self.config_provider = config_provider
-        self.echo = echo
-        self.abort = abort
         self.get_git_remote = get_git_remote
         self.get_codestar_arn = get_codestar_arn
+        self.io = io
 
     def generate(self, terraform_platform_modules_version, deploy_branch):
         pipeline_config = self.config_provider.load_and_validate_platform_config()
@@ -40,7 +39,7 @@ class Pipelines:
         has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
 
         if not (has_codebase_pipelines or has_environment_pipelines):
-            self.echo("No pipelines defined: nothing to do.", err=True, fg="yellow")
+            self.io.warn("No pipelines defined: nothing to do.")
             return
 
         platform_config_terraform_modules_default_version = pipeline_config.get(
@@ -52,11 +51,11 @@ class Pipelines:
 
         git_repo = self.get_git_remote()
         if not git_repo:
-            self.abort("The current directory is not a git repository")
+            self.io.abort_with_error("The current directory is not a git repository")
 
         codestar_connection_arn = self.get_codestar_arn(app_name)
         if codestar_connection_arn is None:
-            self.abort(f'There is no CodeStar Connection named "{app_name}" to use')
+            self.io.abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
 
         base_path = Path(".")
         copilot_pipelines_dir = base_path / f"copilot/pipelines"
@@ -93,7 +92,7 @@ class Pipelines:
 
     def _clean_pipeline_config(self, pipelines_dir):
         if pipelines_dir.exists():
-            self.echo("Deleting copilot/pipelines directory.")
+            self.io.info("Deleting copilot/pipelines directory.")
             rmtree(pipelines_dir)
 
     def _generate_codebase_pipeline(
@@ -152,7 +151,7 @@ class Pipelines:
         message = FileProvider.mkfile(
             base_path, pipelines_dir / file_name, contents, overwrite=True
         )
-        self.echo(message)
+        self.io.info(message)
 
     def _generate_terraform_environment_pipeline_manifest(
         self,
@@ -181,7 +180,7 @@ class Pipelines:
         dir_path = f"terraform/environment-pipelines/{aws_account}"
         makedirs(dir_path, exist_ok=True)
 
-        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
+        self.io.info(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
 
     def generate_terraform_codebase_pipeline_manifest(
         self,
@@ -210,4 +209,4 @@ class Pipelines:
         dir_path = f"terraform/environment-pipelines/{aws_account}"
         makedirs(dir_path, exist_ok=True)
 
-        self.echo(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
+        self.io.info(FileProvider.mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))

dbt_platform_helper/domain/terraform_environment.py

@@ -56,9 +56,9 @@ class TerraformEnvironment:
         self,
         config_provider,
         manifest_generator: PlatformTerraformManifestGenerator = None,
-        echo_fn=click.echo,
+        echo=click.echo,
     ):
-        self.echo = echo_fn
+        self.echo = echo
         self.config_provider = config_provider
         self.manifest_generator = manifest_generator or PlatformTerraformManifestGenerator(
             FileProvider()
@@ -69,7 +69,7 @@ class TerraformEnvironment:
 
         if environment_name not in config.get("environments").keys():
             raise EnvironmentNotFoundException(
-                f"Error: cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+                f"cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
             )
 
         manifest = self.manifest_generator.generate_manifest(

dbt_platform_helper/providers/cloudformation.py

@@ -8,7 +8,8 @@ from dbt_platform_helper.platform_exception import PlatformException
 
 
 class CloudFormation:
-    def __init__(self, cloudformation_client, iam_client, ssm_client):
+    # TODO add handling for optional client parameters to handle case of calling boto API with None
+    def __init__(self, cloudformation_client, iam_client=None, ssm_client=None):
         self.cloudformation_client = cloudformation_client
         self.iam_client = iam_client
         self.ssm_client = ssm_client
@@ -125,6 +126,16 @@ class CloudFormation:
                     stack_name, f"Error while waiting for stack status: {str(err)}"
                 )
 
+    def get_cloudformation_exports_for_environment(self, environment_name):
+        exports = []
+
+        for page in self.cloudformation_client.get_paginator("list_exports").paginate():
+            for export in page["Exports"]:
+                if f"-{environment_name}-" in export["Name"]:
+                    exports.append(export)
+
+        return exports
+
 
 class CloudFormationException(PlatformException):
     def __init__(self, stack_name: str, current_status: str):