dbt-platform-helper 11.0.1__py3-none-any.whl → 11.2.0__py3-none-any.whl

dbt_platform_helper/domain/maintenance_page.py

@@ -0,0 +1,459 @@
+import itertools
+import random
+import re
+import string
+from pathlib import Path
+from typing import List
+from typing import Union
+
+import boto3
+import click
+
+from dbt_platform_helper.providers.load_balancers import ListenerNotFoundError
+from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundError
+from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundError
+from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.utils.application import Environment
+from dbt_platform_helper.utils.application import Service
+from dbt_platform_helper.utils.application import load_application
+
+
+class MaintenancePageProvider:
+    def activate(self, app, env, svc, template, vpc):
+        application = load_application(app)
+        application_environment = get_app_environment(app, env)
+
+        if "*" in svc:
+            services = [
+                s for s in application.services.values() if s.kind == "Load Balanced Web Service"
+            ]
+        else:
+            all_services = [get_app_service(app, s) for s in list(svc)]
+            services = [s for s in all_services if s.kind == "Load Balanced Web Service"]
+
+        if not services:
+            click.secho(f"No services deployed yet to {app} environment {env}", fg="red")
+            raise click.Abort
+
+        try:
+            https_listener = find_https_listener(application_environment.session, app, env)
+            current_maintenance_page = get_maintenance_page(
+                application_environment.session, https_listener
+            )
+            remove_current_maintenance_page = False
+            if current_maintenance_page:
+                remove_current_maintenance_page = click.confirm(
+                    f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
+                    f"environment in {app}.\nWould you like to replace it with a '{template}' "
+                    f"maintenance page?"
+                )
+                if not remove_current_maintenance_page:
+                    raise click.Abort
+
+            if remove_current_maintenance_page or click.confirm(
+                f"You are about to enable the '{template}' maintenance page for the {env} "
+                f"environment in {app}.\nWould you like to continue?"
+            ):
+                if current_maintenance_page and remove_current_maintenance_page:
+                    remove_maintenance_page(application_environment.session, https_listener)
+
+                allowed_ips = get_env_ips(vpc, application_environment)
+
+                add_maintenance_page(
+                    application_environment.session,
+                    https_listener,
+                    app,
+                    env,
+                    services,
+                    allowed_ips,
+                    template,
+                )
+                click.secho(
+                    f"Maintenance page '{template}' added for environment {env} in application {app}",
+                    fg="green",
+                )
+            else:
+                raise click.Abort
+
+        except LoadBalancerNotFoundError:
+            click.secho(
+                f"No load balancer found for environment {env} in the application {app}.", fg="red"
+            )
+            raise click.Abort
+
+        except ListenerNotFoundError:
+            click.secho(
+                f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
+            )
+            raise click.Abort
+
+    def deactivate(self, app, env):
+        application_environment = get_app_environment(app, env)
+
+        try:
+            https_listener = find_https_listener(application_environment.session, app, env)
+            current_maintenance_page = get_maintenance_page(
+                application_environment.session, https_listener
+            )
+            if not current_maintenance_page:
+                click.secho("There is no current maintenance page to remove", fg="red")
+                raise click.Abort
+
+            if not click.confirm(
+                f"There is currently a '{current_maintenance_page}' maintenance page, "
+                f"would you like to remove it?"
+            ):
+                raise click.Abort
+
+            remove_maintenance_page(application_environment.session, https_listener)
+            click.secho(
+                f"Maintenance page removed from environment {env} in application {app}", fg="green"
+            )
+
+        except LoadBalancerNotFoundError:
+            click.secho(
+                f"No load balancer found for environment {env} in the application {app}.", fg="red"
+            )
+            raise click.Abort
+
+        except ListenerNotFoundError:
+            click.secho(
+                f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
+            )
+            raise click.Abort
+
+
+def get_app_service(app_name: str, svc_name: str) -> Service:
+    application = load_application(app_name)
+    application_service = application.services.get(svc_name)
+
+    if not application_service:
+        click.secho(
+            f"The service {svc_name} was not found in the application {app_name}. "
+            f"It either does not exist, or has not been deployed.",
+            fg="red",
+        )
+        raise click.Abort
+
+    return application_service
+
+
+def get_app_environment(app_name: str, env_name: str) -> Environment:
+    application = load_application(app_name)
+    application_environment = application.environments.get(env_name)
+
+    if not application_environment:
+        click.secho(
+            f"The environment {env_name} was not found in the application {app_name}. "
+            f"It either does not exist, or has not been deployed.",
+            fg="red",
+        )
+        raise click.Abort
+
+    return application_environment
+
+
+def get_maintenance_page(session: boto3.Session, listener_arn: str) -> Union[str, None]:
+    lb_client = session.client("elbv2")
+
+    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
+    tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
+
+    maintenance_page_type = None
+    for description in tag_descriptions:
+        tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+        if tags.get("name") == "MaintenancePage":
+            maintenance_page_type = tags.get("type")
+
+    return maintenance_page_type
+
+
+def remove_maintenance_page(session: boto3.Session, listener_arn: str):
+    lb_client = session.client("elbv2")
+
+    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
+    # TODO: The next line doesn't appear to do anything.
+    tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
+    # TODO: In fact the following line seems to do the same but better.
+    tag_descriptions = lb_client.describe_tags(ResourceArns=[r["RuleArn"] for r in rules])[
+        "TagDescriptions"
+    ]
+
+    for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
+        deleted = delete_listener_rule(tag_descriptions, name, lb_client)
+
+        if name == "MaintenancePage" and not deleted:
+            raise ListenerRuleNotFoundError()
+
+
+def get_rules_tag_descriptions(rules: list, lb_client):
+    tag_descriptions = []
+    chunk_size = 20
+
+    for i in range(0, len(rules), chunk_size):
+        chunk = rules[i : i + chunk_size]
+        resource_arns = [r["RuleArn"] for r in chunk]
+        response = lb_client.describe_tags(ResourceArns=resource_arns)
+        tag_descriptions.extend(response["TagDescriptions"])
+
+    return tag_descriptions
+
+
+def delete_listener_rule(tag_descriptions: list, tag_name: str, lb_client: boto3.client):
+    current_rule_arn = None
+
+    for description in tag_descriptions:
+        tags = {t["Key"]: t["Value"] for t in description["Tags"]}
+        if tags.get("name") == tag_name:
+            current_rule_arn = description["ResourceArn"]
+            if current_rule_arn:
+                lb_client.delete_rule(RuleArn=current_rule_arn)
+
+    return current_rule_arn
+
+
+def add_maintenance_page(
+    session: boto3.Session,
+    listener_arn: str,
+    app: str,
+    env: str,
+    services: List[Service],
+    allowed_ips: tuple,
+    template: str = "default",
+):
+    lb_client = session.client("elbv2")
+    maintenance_page_content = get_maintenance_page_template(template)
+    bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
+
+    rule_priority = itertools.count(start=1)
+
+    for svc in services:
+        target_group_arn = find_target_group(app, env, svc.name, session)
+
+        # not all of an application's services are guaranteed to have been deployed to an environment
+        if not target_group_arn:
+            continue
+
+        for ip in allowed_ips:
+            create_header_rule(
+                lb_client,
+                listener_arn,
+                target_group_arn,
+                "X-Forwarded-For",
+                [ip],
+                "AllowedIps",
+                next(rule_priority),
+            )
+            create_source_ip_rule(
+                lb_client,
+                listener_arn,
+                target_group_arn,
+                [ip],
+                "AllowedSourceIps",
+                next(rule_priority),
+            )
+
+        create_header_rule(
+            lb_client,
+            listener_arn,
+            target_group_arn,
+            "Bypass-Key",
+            [bypass_value],
+            "BypassIpFilter",
+            next(rule_priority),
+        )
+
+        click.secho(
+            f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/activities/holding-and-maintenance-pages/",
+            fg="green",
+        )
+
+    lb_client.create_rule(
+        ListenerArn=listener_arn,
+        Priority=next(rule_priority),
+        Conditions=[
+            {
+                "Field": "path-pattern",
+                "PathPatternConfig": {"Values": ["/*"]},
+            }
+        ],
+        Actions=[
+            {
+                "Type": "fixed-response",
+                "FixedResponseConfig": {
+                    "StatusCode": "503",
+                    "ContentType": "text/html",
+                    "MessageBody": maintenance_page_content,
+                },
+            }
+        ],
+        Tags=[
+            {"Key": "name", "Value": "MaintenancePage"},
+            {"Key": "type", "Value": template},
+        ],
+    )
+
+
+def get_maintenance_page_template(template) -> str:
+    template_contents = (
+        Path(__file__)
+        .parent.parent.joinpath(
+            f"templates/svc/maintenance_pages/{template}.html",
+        )
+        .read_text()
+        .replace("\n", "")
+    )
+
+    # [^\S]\s+ - Remove any space that is not preceded by a non-space character.
+    return re.sub(r"[^\S]\s+", "", template_contents)
+
+
+def find_target_group(app: str, env: str, svc: str, session: boto3.Session) -> str:
+    rg_tagging_client = session.client("resourcegroupstaggingapi")
+    response = rg_tagging_client.get_resources(
+        TagFilters=[
+            {
+                "Key": "copilot-application",
+                "Values": [
+                    app,
+                ],
+                "Key": "copilot-environment",
+                "Values": [
+                    env,
+                ],
+                "Key": "copilot-service",
+                "Values": [
+                    svc,
+                ],
+            },
+        ],
+        ResourceTypeFilters=[
+            "elasticloadbalancing:targetgroup",
+        ],
+    )
+    for resource in response["ResourceTagMappingList"]:
+        tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
+
+        if (
+            "copilot-service" in tags
+            and tags["copilot-service"] == svc
+            and "copilot-environment" in tags
+            and tags["copilot-environment"] == env
+            and "copilot-application" in tags
+            and tags["copilot-application"] == app
+        ):
+            return resource["ResourceARN"]
+
+    click.secho(
+        f"No target group found for application: {app}, environment: {env}, service: {svc}",
+        fg="red",
+    )
+
+    return None
+
+
+def create_header_rule(
+    lb_client: boto3.client,
+    listener_arn: str,
+    target_group_arn: str,
+    header_name: str,
+    values: list,
+    rule_name: str,
+    priority: int,
+):
+    conditions = get_host_conditions(lb_client, listener_arn, target_group_arn)
+
+    # add new condition to existing conditions
+    combined_conditions = [
+        {
+            "Field": "http-header",
+            "HttpHeaderConfig": {"HttpHeaderName": header_name, "Values": values},
+        }
+    ] + conditions
+
+    lb_client.create_rule(
+        ListenerArn=listener_arn,
+        Priority=priority,
+        Conditions=combined_conditions,
+        Actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+        Tags=[
+            {"Key": "name", "Value": rule_name},
+        ],
+    )
+
+    click.secho(
+        f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        fg="green",
+    )
+
+
+def create_source_ip_rule(
+    lb_client: boto3.client,
+    listener_arn: str,
+    target_group_arn: str,
+    values: list,
+    rule_name: str,
+    priority: int,
+):
+    conditions = get_host_conditions(lb_client, listener_arn, target_group_arn)
+
+    # add new condition to existing conditions
+    combined_conditions = [
+        {
+            "Field": "source-ip",
+            "SourceIpConfig": {"Values": [value + "/32" for value in values]},
+        }
+    ] + conditions
+
+    lb_client.create_rule(
+        ListenerArn=listener_arn,
+        Priority=priority,
+        Conditions=combined_conditions,
+        Actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
+        Tags=[
+            {"Key": "name", "Value": rule_name},
+        ],
+    )
+
+    click.secho(
+        f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
+        fg="green",
+    )
+
+
+def get_host_conditions(lb_client: boto3.client, listener_arn: str, target_group_arn: str):
+    rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
+
+    # Get current set of forwarding conditions for the target group
+    for rule in rules:
+        for action in rule["Actions"]:
+            if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
+                conditions = rule["Conditions"]
+
+    # filter to host-header conditions
+    conditions = [
+        {i: condition[i] for i in condition if i != "Values"}
+        for condition in conditions
+        if condition["Field"] == "host-header"
+    ]
+
+    # remove internal hosts
+    conditions[0]["HostHeaderConfig"]["Values"] = [
+        v for v in conditions[0]["HostHeaderConfig"]["Values"]
+    ]
+
+    return conditions
+
+
+def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
+    account_name = f"{application_environment.session.profile_name}-vpc"
+    vpc_name = vpc if vpc else account_name
+    ssm_client = application_environment.session.client("ssm")
+
+    try:
+        param_value = ssm_client.get_parameter(Name=f"/{vpc_name}/EGRESS_IPS")["Parameter"]["Value"]
+    except ssm_client.exceptions.ParameterNotFound:
+        click.secho(f"No parameter found with name: /{vpc_name}/EGRESS_IPS")
+        raise click.Abort
+
+    return [ip.strip() for ip in param_value.split(",")]

dbt_platform_helper/providers/load_balancers.py

@@ -0,0 +1,51 @@
+import boto3
+
+
+def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
+    lb_client = session.client("elbv2")
+
+    describe_response = lb_client.describe_load_balancers()
+    load_balancers = [lb["LoadBalancerArn"] for lb in describe_response["LoadBalancers"]]
+
+    load_balancers = lb_client.describe_tags(ResourceArns=load_balancers)["TagDescriptions"]
+
+    load_balancer_arn = None
+    for lb in load_balancers:
+        tags = {t["Key"]: t["Value"] for t in lb["Tags"]}
+        if tags.get("copilot-application") == app and tags.get("copilot-environment") == env:
+            load_balancer_arn = lb["ResourceArn"]
+
+    if not load_balancer_arn:
+        raise LoadBalancerNotFoundError()
+
+    return load_balancer_arn
+
+
+def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
+    load_balancer_arn = find_load_balancer(session, app, env)
+    lb_client = session.client("elbv2")
+    listeners = lb_client.describe_listeners(LoadBalancerArn=load_balancer_arn)["Listeners"]
+
+    listener_arn = None
+
+    try:
+        listener_arn = next(l["ListenerArn"] for l in listeners if l["Protocol"] == "HTTPS")
+    except StopIteration:
+        pass
+
+    if not listener_arn:
+        raise ListenerNotFoundError()
+
+    return listener_arn
+
+
+class LoadBalancerNotFoundError(Exception):
+    pass
+
+
+class ListenerNotFoundError(Exception):
+    pass
+
+
+class ListenerRuleNotFoundError(Exception):
+    pass

dbt_platform_helper/templates/environment-pipelines/main.tf

@@ -0,0 +1,52 @@
+# {% extra_header %}
+# {% version_info %}
+locals {
+  platform_config    = yamldecode(file("../../../platform-config.yml"))
+  all_pipelines      = local.platform_config["environment_pipelines"]
+  pipelines          = { for pipeline, config in local.platform_config["environment_pipelines"] : pipeline => config if config.account == "{{ aws_account }}" }
+  environment_config = local.platform_config["environments"]
+}
+
+provider "aws" {
+  region                   = "eu-west-2"
+  profile                  = "{{ aws_account }}"
+  alias                    = "{{ aws_account }}"
+  shared_credentials_files = ["~/.aws/config"]
+}
+
+terraform {
+  required_version = "~> 1.8"
+  backend "s3" {
+    bucket         = "terraform-platform-state-{{ aws_account }}"
+    key            = "tfstate/application/{{ application }}-pipelines.tfstate"
+    region         = "eu-west-2"
+    encrypt        = true
+    kms_key_id     = "alias/terraform-platform-state-s3-key-{{ aws_account }}"
+    dynamodb_table = "terraform-platform-lockdb-{{ aws_account }}"
+  }
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5"
+    }
+  }
+}
+
+
+module "environment-pipelines" {
+  source = "git::https://github.com/uktrade/terraform-platform-modules.git//environment-pipelines?depth=1&ref={{ terraform_platform_modules_version }}"
+
+  for_each = local.pipelines
+
+  application         = "{{ application }}"
+  pipeline_name       = each.key
+  repository          = "uktrade/{{ application }}-deploy"
+
+  environments        = each.value.environments
+  all_pipelines       = local.all_pipelines
+  environment_config  = local.environment_config
+  branch              = {% if deploy_branch %}"{{ deploy_branch }}"{% else %}each.value.branch{% endif %}
+  slack_channel       = each.value.slack_channel
+  trigger_on_push     = each.value.trigger_on_push
+  pipeline_to_trigger = lookup(each.value, "pipeline_to_trigger", null)
+}

dbt_platform_helper/utils/aws.py

@@ -7,6 +7,7 @@ from typing import Tuple
 
 import boto3
 import botocore
+import botocore.exceptions
 import click
 import yaml
 from boto3 import Session
@@ -20,62 +21,71 @@ AWS_SESSION_CACHE = {}
 
 
 def get_aws_session_or_abort(aws_profile: str = None) -> boto3.session.Session:
-    aws_profile = aws_profile if aws_profile else os.getenv("AWS_PROFILE")
+    REFRESH_TOKEN_MESSAGE = (
+        "To refresh this SSO session run `aws sso login` with the corresponding profile"
+    )
+    aws_profile = aws_profile or os.getenv("AWS_PROFILE")
     if aws_profile in AWS_SESSION_CACHE:
         return AWS_SESSION_CACHE[aws_profile]
 
-    # Check that the aws profile exists and is set.
-    click.secho(f"""Checking AWS connection for profile "{aws_profile}"...""", fg="cyan")
+    click.secho(f'Checking AWS connection for profile "{aws_profile}"...', fg="cyan")
 
     try:
         session = boto3.session.Session(profile_name=aws_profile)
+        sts = session.client("sts")
+        account_id, user_id = get_account_details(sts)
+        click.secho("Credentials are valid.", fg="green")
+
     except botocore.exceptions.ProfileNotFound:
-        click.secho(f"""AWS profile "{aws_profile}" is not configured.""", fg="red")
-        exit(1)
+        _handle_error(f'AWS profile "{aws_profile}" is not configured.')
     except botocore.exceptions.ClientError as e:
         if e.response["Error"]["Code"] == "ExpiredToken":
-            click.secho(
-                f"Credentials are NOT valid.  \nPlease login with: aws sso login --profile {aws_profile}",
-                fg="red",
+            _handle_error(
+                f"Credentials are NOT valid.  \nPlease login with: aws sso login --profile {aws_profile}"
             )
-            exit(1)
-
-    sts = session.client("sts")
-    try:
-        account_id, user_id = get_account_details(sts)
-        click.secho("Credentials are valid.", fg="green")
-    except (
-        botocore.exceptions.UnauthorizedSSOTokenError,
-        botocore.exceptions.TokenRetrievalError,
-        botocore.exceptions.SSOTokenLoadError,
-    ):
-        click.secho(
-            "The SSO session associated with this profile has expired or is otherwise invalid."
-            "To refresh this SSO session run `aws sso login` with the corresponding profile",
-            fg="red",
+    except botocore.exceptions.NoCredentialsError:
+        _handle_error("There are no credentials set for this session.", REFRESH_TOKEN_MESSAGE)
+    except botocore.exceptions.UnauthorizedSSOTokenError:
+        _handle_error("The SSO Token used for this session is unauthorised.", REFRESH_TOKEN_MESSAGE)
+    except botocore.exceptions.TokenRetrievalError:
+        _handle_error("Unable to retrieve the Token for this session.", REFRESH_TOKEN_MESSAGE)
+    except botocore.exceptions.SSOTokenLoadError:
+        _handle_error(
+            "The SSO session associated with this profile has expired, is not set or is otherwise invalid.",
+            REFRESH_TOKEN_MESSAGE,
         )
-        exit(1)
 
     alias_client = session.client("iam")
-    account_name = alias_client.list_account_aliases()["AccountAliases"]
+    account_name = alias_client.list_account_aliases().get("AccountAliases", [])
+
+    _log_account_info(account_name, account_id)
+
+    click.echo(
+        click.style("User: ", fg="yellow")
+        + click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True)
+    )
+
+    AWS_SESSION_CACHE[aws_profile] = session
+    return session
+
+
+def _handle_error(message: str, refresh_token_message: str = None) -> None:
+    full_message = message + (" " + refresh_token_message if refresh_token_message else "")
+    click.secho(full_message, fg="red")
+    exit(1)
+
+
+def _log_account_info(account_name: list, account_id: str) -> None:
     if account_name:
         click.echo(
             click.style("Logged in with AWS account: ", fg="yellow")
-            + click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True),
+            + click.style(f"{account_name[0]}/{account_id}", fg="white", bold=True)
         )
     else:
         click.echo(
             click.style("Logged in with AWS account id: ", fg="yellow")
-            + click.style(f"{account_id}", fg="white", bold=True),
+            + click.style(f"{account_id}", fg="white", bold=True)
         )
-    click.echo(
-        click.style("User: ", fg="yellow")
-        + click.style(f"{user_id.split(':')[-1]}\n", fg="white", bold=True),
-    )
-
-    AWS_SESSION_CACHE[aws_profile] = session
-
-    return session
 
 
 class NoProfileForAccountIdError(Exception):
@@ -362,12 +372,12 @@ def get_connection_string(
 
 
 class Vpc:
-    def __init__(self, subnets, security_groups):
+    def __init__(self, subnets: list[str], security_groups: list[str]):
         self.subnets = subnets
         self.security_groups = security_groups
 
 
-def get_vpc_info_by_name(session, app, env, vpc_name):
+def get_vpc_info_by_name(session: Session, app: str, env: str, vpc_name: str) -> Vpc:
     ec2_client = session.client("ec2")
     vpc_response = ec2_client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [vpc_name]}])