dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.0__py3-none-any.whl

dbt_platform_helper/COMMANDS.md

@@ -551,9 +551,6 @@ platform-helper pipeline generate
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
 
-    (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
-    - Generates the copilot manifest.yml for copilot/environments/<environment>
-
 ## Usage
 
 ```

dbt_platform_helper/commands/config.py

@@ -8,11 +8,11 @@ import botocore
 import click
 from prettytable import PrettyTable
 
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
 from dbt_platform_helper.providers.validation import ValidationException
 from dbt_platform_helper.utils import versioning
 from dbt_platform_helper.utils.click import ClickDocOptGroup
-from dbt_platform_helper.utils.validation import config_file_check
 from dbt_platform_helper.utils.versioning import get_platform_helper_versions
 
 yes = "\033[92m✔\033[0m"
@@ -98,7 +98,7 @@ def deployment():
 
     recommendations = {}
 
-    config_file_check()
+    ConfigProvider.config_file_check()
 
     for template_file in addons_templates:
         generated_with_version = maybe

dbt_platform_helper/commands/copilot.py

@@ -7,17 +7,22 @@ from pathlib import PosixPath
 
 import click
 import yaml
+from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.files import generate_override_files
 from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.messages import abort_with_error
+from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.validation import config_file_check
 from dbt_platform_helper.utils.validation import validate_addons
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
@@ -35,14 +40,6 @@ SERVICE_TYPES = [
     "Worker Service",
 ]
 
-ADDON_TEMPLATE_MAP = {
-    "s3": ["addons/svc/s3-policy.yml"],
-    "s3-policy": ["addons/svc/s3-policy.yml"],
-    "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
-    "subscription-filter": ["addons/svc/subscription-filter.yml"],
-    "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
-}
-
 
 def list_copilot_local_environments():
     return [
@@ -244,25 +241,42 @@ def _get_s3_kms_alias_arns(session, application_name, config):
     return arns
 
 
+def copilot_provider():
+    return CopilotTemplating()
+
+
 @copilot.command()
 def make_addons():
     """Generate addons CloudFormation for each environment."""
-    output_dir = Path(".").absolute()
-    config_file_check()
+    try:
+        config_provider = ConfigProvider(ConfigValidator())
+        _make_addons(config_provider)
+    except Exception as exc:
+        abort_with_error(exc)
+
+
+def _make_addons(config_provider: ConfigProvider):
+    config_provider.config_file_check()
+    try:
+        config = config_provider.load_and_validate_platform_config()
+    except SchemaError as ex:
+        click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
+        raise click.Abort
 
     templates = setup_templates()
-    config = _get_config()
+    extensions = _get_extensions()
     session = get_aws_session_or_abort()
 
     application_name = get_application_name()
 
     click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
 
+    output_dir = Path(".").absolute()
     env_path = Path(f"copilot/environments/")
     env_addons_path = env_path / "addons"
     env_overrides_path = env_path / "overrides"
 
-    _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path)
+    _cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
     _generate_env_overrides(output_dir)
 
     svc_names = list_copilot_local_services()
@@ -271,33 +285,33 @@ def make_addons():
         _generate_svc_overrides(base_path, templates, svc_name)
 
     services = []
-    for addon_name, addon_config in config.items():
-        print(f">>>>>>>>> {addon_name}")
-        addon_type = addon_config.pop("type")
-        environments = addon_config.pop("environments")
+    for ext_name, ext_data in extensions.items():
+        extension = {**ext_data}
+        addon_type = extension.pop("type")
+        environments = extension.pop("environments")
         environment_addon_config = {
             "addon_type": addon_type,
             "environments": environments,
-            "name": addon_config.get("name", None) or addon_name,
-            "prefix": camel_case(addon_name),
-            "secret_name": addon_name.upper().replace("-", "_"),
-            **addon_config,
+            "name": extension.get("name", None) or ext_name,
+            "prefix": camel_case(ext_name),
+            "secret_name": ext_name.upper().replace("-", "_"),
+            **extension,
         }
 
         services.append(environment_addon_config)
 
         service_addon_config = {
             "application_name": application_name,
-            "name": addon_config.get("name", None) or addon_name,
-            "prefix": camel_case(addon_name),
+            "name": extension.get("name", None) or ext_name,
+            "prefix": camel_case(ext_name),
             "environments": environments,
-            **addon_config,
+            **extension,
         }
 
         log_destination_arns = get_log_destination_arn()
 
         if addon_type in ["s3", "s3-policy"]:
-            if config[addon_name].get("serve_static_content"):
+            if extensions[ext_name].get("serve_static_content"):
                 continue
 
             s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
@@ -307,8 +321,8 @@ def make_addons():
                 )
 
         _generate_service_addons(
-            addon_config,
-            addon_name,
+            extension,
+            ext_name,
             addon_type,
             output_dir,
             service_addon_config,
@@ -316,10 +330,15 @@ def make_addons():
             log_destination_arns,
         )
 
+    environments = config_provider.apply_environment_defaults(config)["environments"]
+
+    provider = copilot_provider()
+    provider.generate_cross_account_s3_policies(environments, extensions)
+
     click.echo(templates.get_template("addon-instructions.txt").render(services=services))
 
 
-def _get_config():
+def _get_extensions():
     config = _validate_and_normalise_extensions_config(PACKAGE_DIR / "default-extensions.yml")
     project_config = _validate_and_normalise_extensions_config(PLATFORM_CONFIG_FILE, "extensions")
     config.update(project_config)

dbt_platform_helper/commands/environment.py

@@ -1,18 +1,15 @@
-import boto3
 import click
 from schema import SchemaError
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
 from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
 from dbt_platform_helper.platform_exception import PlatformException
-from dbt_platform_helper.providers.load_balancers import find_https_listener
-from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.click import ClickDocOptGroup
-from dbt_platform_helper.utils.files import apply_environment_defaults
-from dbt_platform_helper.utils.files import mkfile
-from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.validation import load_and_validate_platform_config
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
@@ -39,8 +36,11 @@ def environment():
 @click.option("--vpc", type=str)
 def offline(app, env, svc, template, vpc):
     """Take load-balanced web services offline with a maintenance page."""
-    maintenance_page = MaintenancePageProvider()
-    maintenance_page.activate(app, env, svc, template, vpc)
+    try:
+        MaintenancePageProvider().activate(app, env, svc, template, vpc)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
+        raise click.Abort
 
 
 @environment.command()
@@ -48,88 +48,12 @@ def offline(app, env, svc, template, vpc):
 @click.option("--env", type=str, required=True)
 def online(app, env):
     """Remove a maintenance page from an environment."""
-    maintenance_page = MaintenancePageProvider()
-    maintenance_page.deactivate(app, env)
-
-
-def get_vpc_id(session, env_name, vpc_name=None):
-    if not vpc_name:
-        vpc_name = f"{session.profile_name}-{env_name}"
-
-    filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
-    vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
-
-    if not vpcs:
-        filters[0]["Values"] = [session.profile_name]
-        vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
-
-    if not vpcs:
-        click.secho(
-            f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
-        )
-        raise click.Abort
-
-    return vpcs[0]["VpcId"]
-
-
-def get_subnet_ids(session, vpc_id, environment_name):
-    subnets = session.client("ec2").describe_subnets(
-        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
-    )["Subnets"]
-
-    if not subnets:
-        click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
-        raise click.Abort
-
-    public_tag = {"Key": "subnet_type", "Value": "public"}
-    public_subnets = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
-    private_tag = {"Key": "subnet_type", "Value": "private"}
-    private_subnets = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
-
-    # This call and the method declaration can be removed when we stop using AWS Copilot to deploy the services
-    public_subnets, private_subnets = _match_subnet_id_order_to_cloudformation_exports(
-        session,
-        environment_name,
-        public_subnets,
-        private_subnets,
-    )
-
-    return public_subnets, private_subnets
-
-
-def _match_subnet_id_order_to_cloudformation_exports(
-    session, environment_name, public_subnets, private_subnets
-):
-    public_subnet_exports = []
-    private_subnet_exports = []
-    for page in session.client("cloudformation").get_paginator("list_exports").paginate():
-        for export in page["Exports"]:
-            if f"-{environment_name}-" in export["Name"]:
-                if export["Name"].endswith("-PublicSubnets"):
-                    public_subnet_exports = export["Value"].split(",")
-                if export["Name"].endswith("-PrivateSubnets"):
-                    private_subnet_exports = export["Value"].split(",")
-
-    # If the elements match, regardless of order, use the list from the CloudFormation exports
-    if set(public_subnets) == set(public_subnet_exports):
-        public_subnets = public_subnet_exports
-    if set(private_subnets) == set(private_subnet_exports):
-        private_subnets = private_subnet_exports
-
-    return public_subnets, private_subnets
-
-
-def get_cert_arn(session, application, env_name):
     try:
-        arn = find_https_certificate(session, application, env_name)
-    except:
-        click.secho(
-            f"No certificate found with domain name matching environment {env_name}.", fg="red"
-        )
+        MaintenancePageProvider().deactivate(app, env)
+    except PlatformException as err:
+        click.secho(str(err), fg="red")
         raise click.Abort
 
-    return arn
-
 
 @environment.command()
 @click.option("--vpc-name", hidden=True)
@@ -141,20 +65,13 @@ def generate(name, vpc_name):
             fg="red",
         )
         raise click.Abort
-
     try:
-        conf = load_and_validate_platform_config()
+        config_provider = ConfigProvider(ConfigValidator())
+        CopilotEnvironment(config_provider).generate(name)
     except SchemaError as ex:
         click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
         raise click.Abort
 
-    env_config = apply_environment_defaults(conf)["environments"][name]
-    profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
-    click.secho(f"Using {profile_for_environment} for this AWS session")
-    session = get_aws_session_or_abort(profile_for_environment)
-
-    _generate_copilot_environment_manifests(name, conf["application"], env_config, session)
-
 
 @environment.command(help="Generate terraform manifest for the specified environment.")
 @click.option(
@@ -165,84 +82,5 @@ def generate(name, vpc_name):
     help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
 )
 def generate_terraform(name, terraform_platform_modules_version):
-    conf = load_and_validate_platform_config()
-
-    env_config = apply_environment_defaults(conf)["environments"][name]
-    _generate_terraform_environment_manifests(
-        conf["application"], name, env_config, terraform_platform_modules_version
-    )
-
-
-def _generate_copilot_environment_manifests(environment_name, application, env_config, session):
-    env_template = setup_templates().get_template("env/manifest.yml")
-    vpc_name = env_config.get("vpc", None)
-    vpc_id = get_vpc_id(session, environment_name, vpc_name)
-    pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id, environment_name)
-    cert_arn = get_cert_arn(session, application, environment_name)
-    contents = env_template.render(
-        {
-            "name": environment_name,
-            "vpc_id": vpc_id,
-            "pub_subnet_ids": pub_subnet_ids,
-            "priv_subnet_ids": priv_subnet_ids,
-            "certificate_arn": cert_arn,
-        }
-    )
-    click.echo(
-        mkfile(
-            ".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
-        )
-    )
-
-
-def _generate_terraform_environment_manifests(
-    application, env, env_config, cli_terraform_platform_modules_version
-):
-    env_template = setup_templates().get_template("environments/main.tf")
-
-    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
-        env_config, cli_terraform_platform_modules_version
-    )
-
-    contents = env_template.render(
-        {
-            "application": application,
-            "environment": env,
-            "config": env_config,
-            "terraform_platform_modules_version": terraform_platform_modules_version,
-        }
-    )
-
-    click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
-
-
-def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
-    cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
-    env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
-        "terraform-platform-modules"
-    )
-    version_preference_order = [
-        cli_terraform_platform_modules_version,
-        env_conf_terraform_platform_modules_version,
-        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
-    ]
-    return [version for version in version_preference_order if version][0]
-
-
-def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
-    listener_arn = find_https_listener(session, app, env)
-    cert_client = session.client("elbv2")
-    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
-        "Certificates"
-    ]
-
-    try:
-        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
-    except StopIteration:
-        raise CertificateNotFoundException()
-
-    return certificate_arn
-
-
-class CertificateNotFoundException(PlatformException):
-    pass
+    config_provider = ConfigProvider(ConfigValidator())
+    TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)

dbt_platform_helper/commands/pipeline.py

@@ -6,6 +6,8 @@ from shutil import rmtree
 import click
 
 from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.aws import get_account_details
 from dbt_platform_helper.utils.aws import get_codestar_connection_arn
@@ -16,7 +18,6 @@ from dbt_platform_helper.utils.files import mkfile
 from dbt_platform_helper.utils.git import git_remote
 from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.validation import load_and_validate_platform_config
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
@@ -60,11 +61,9 @@ def generate(terraform_platform_modules_version, deploy_branch):
 
     This command does the following in relation to the codebase pipelines:
     - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
-
-    (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
-    - Generates the copilot manifest.yml for copilot/environments/<environment>
     """
-    pipeline_config = load_and_validate_platform_config()
+    config_provider = ConfigProvider(ConfigValidator())
+    pipeline_config = config_provider.load_and_validate_platform_config()
 
     has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
     has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config

dbt_platform_helper/constants.py

@@ -16,3 +16,12 @@ CONDUIT_ADDON_TYPES = [
     "redis",
 ]
 CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"
+HYPHENATED_APPLICATION_NAME = "hyphenated-application-name"
+ALPHANUMERIC_ENVIRONMENT_NAME = "alphanumericenvironmentname123"
+ALPHANUMERIC_SERVICE_NAME = "alphanumericservicename123"
+COPILOT_IDENTIFIER = "c0PIlotiD3ntIF3r"
+CLUSTER_NAME_SUFFIX = f"Cluster-{COPILOT_IDENTIFIER}"
+SERVICE_NAME_SUFFIX = f"Service-{COPILOT_IDENTIFIER}"
+REFRESH_TOKEN_MESSAGE = (
+    "To refresh this SSO session run `aws sso login` with the corresponding profile"
+)