dbt-platform-helper 13.0.0__tar.gz → 13.0.2__tar.gz

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 13.0.0
+Version: 13.0.2
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/COMMANDS.md

@@ -856,8 +856,8 @@ platform-helper database copy --from <from_env> --to <to_env> --database <databa
 
 [↩ Parent](#platform-helper)
 
-    Contains subcommands for getting version information about the
-        current project.
+    Contains subcommands for getting version information about the current
+    project.
 
 ## Usage
 

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/commands/config.py

@@ -9,7 +9,9 @@ import click
 from prettytable import PrettyTable
 
 from dbt_platform_helper.providers.config import ConfigProvider
-from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMajorVersionException,
+)
 from dbt_platform_helper.providers.validation import ValidationException
 from dbt_platform_helper.utils import versioning
 from dbt_platform_helper.utils.click import ClickDocOptGroup
@@ -79,8 +81,8 @@ def deployment():
     _check_tool_versions(platform_helper_versions, copilot_versions, aws_versions)
     click.secho("Checking addons templates versions...", fg="blue")
 
-    local_version = platform_helper_versions.local_version
-    latest_release = platform_helper_versions.latest_release
+    local_version = platform_helper_versions.local
+    latest_release = platform_helper_versions.latest
     addons_templates_table = PrettyTable()
     addons_templates_table.field_names = [
         "Addons Template File",
@@ -115,7 +117,7 @@ def deployment():
             compatible = False
             recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
                 "dbt-platform-helper-upgrade"
-            ].format(version=versioning.string_version(latest_release))
+            ].format(version=latest_release)
             recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
                 "dbt-platform-helper-upgrade-note"
             ]
@@ -124,7 +126,7 @@ def deployment():
             compatible = False
             recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
                 "dbt-platform-helper-upgrade"
-            ].format(version=versioning.string_version(latest_release))
+            ].format(version=latest_release)
             recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
                 "dbt-platform-helper-upgrade-note"
             ]
@@ -144,11 +146,7 @@ def deployment():
         addons_templates_table.add_row(
             [
                 template_file.relative_to("."),
-                (
-                    maybe
-                    if latest_compatible_symbol is maybe
-                    else versioning.string_version(generated_with_version)
-                ),
+                (maybe if latest_compatible_symbol is maybe else str(generated_with_version)),
                 local_compatible_symbol,
                 latest_compatible_symbol,
             ]
@@ -164,21 +162,16 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
     click.secho("Checking tooling versions...", fg="blue")
     recommendations = {}
 
-    local_copilot_version = copilot_versions.local_version
-    copilot_latest_release = copilot_versions.latest_release
+    local_copilot_version = copilot_versions.local
+    copilot_latest_release = copilot_versions.latest
     if local_copilot_version is None:
         recommendations["install-copilot"] = (
             "Install AWS Copilot https://aws.github.io/copilot-cli/"
         )
 
-    local_aws_version = aws_versions.local_version
-    aws_latest_release = aws_versions.latest_release
-    if local_aws_version is None:
+    if aws_versions.local is None:
         recommendations["install-aws"] = "Install AWS CLI https://aws.amazon.com/cli/"
 
-    local_version = platform_helper_versions.local_version
-    latest_release = platform_helper_versions.latest_release
-
     tool_versions_table = PrettyTable()
     tool_versions_table.field_names = [
         "Tool",
@@ -191,46 +184,46 @@ def _check_tool_versions(platform_helper_versions, copilot_versions, aws_version
     tool_versions_table.add_row(
         [
             "aws",
-            versioning.string_version(local_aws_version),
-            versioning.string_version(aws_latest_release),
-            no if local_aws_version != aws_latest_release else yes,
+            str(aws_versions.local),
+            str(aws_versions.latest),
+            no if aws_versions.is_outdated() else yes,
         ]
     )
     tool_versions_table.add_row(
         [
             "copilot",
-            versioning.string_version(local_copilot_version),
-            versioning.string_version(copilot_latest_release),
-            no if local_copilot_version != copilot_latest_release else yes,
+            str(copilot_versions.local),
+            str(copilot_versions.latest),
+            no if copilot_versions.is_outdated() else yes,
         ]
     )
     tool_versions_table.add_row(
         [
             "dbt-platform-helper",
-            versioning.string_version(local_version),
-            versioning.string_version(latest_release),
-            no if local_version != latest_release else yes,
+            str(platform_helper_versions.local),
+            str(platform_helper_versions.latest),
+            no if platform_helper_versions.is_outdated() else yes,
         ]
     )
 
     click.secho(tool_versions_table)
 
-    if local_aws_version != aws_latest_release and "install-aws" not in recommendations:
+    if aws_versions.is_outdated() and "install-aws" not in recommendations:
         recommendations["aws-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
             tool="AWS CLI",
-            version=versioning.string_version(aws_latest_release),
+            version=str(aws_versions.latest),
         )
 
-    if local_copilot_version != copilot_latest_release and "install-copilot" not in recommendations:
+    if copilot_versions.is_outdated() and "install-copilot" not in recommendations:
         recommendations["copilot-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
             tool="AWS Copilot",
-            version=versioning.string_version(copilot_latest_release),
+            version=str(copilot_latest_release),
         )
 
-    if local_version != latest_release:
+    if platform_helper_versions.is_outdated():
         recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
             "dbt-platform-helper-upgrade"
-        ].format(version=versioning.string_version(latest_release))
+        ].format(version=str(platform_helper_versions.latest))
         recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
             "dbt-platform-helper-upgrade-note"
         ]

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/commands/generate.py

@@ -4,7 +4,7 @@ import click
 from dbt_platform_helper.commands.copilot import make_addons
 from dbt_platform_helper.commands.pipeline import generate as pipeline_generate
 from dbt_platform_helper.utils.click import ClickDocOptCommand
-from dbt_platform_helper.utils.versioning import check_platform_helper_version_mismatch
+from dbt_platform_helper.utils.versioning import RequiredVersion
 
 
 @click.command(cls=ClickDocOptCommand)
@@ -17,6 +17,6 @@ def generate(ctx: click.Context):
     Wraps pipeline generate and make-addons.
     """
 
-    check_platform_helper_version_mismatch()
+    RequiredVersion().check_platform_helper_version_mismatch()
     ctx.invoke(pipeline_generate)
     ctx.invoke(make_addons)

dbt_platform_helper-13.0.2/dbt_platform_helper/commands/version.py

@@ -0,0 +1,37 @@
+import click
+
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.versioning import RequiredVersion
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def version():
+    """Contains subcommands for getting version information about the current
+    project."""
+
+
+@version.command(help="Print the version of platform-tools required by the current project")
+@click.option(
+    "--pipeline",
+    required=False,
+    type=str,
+    help="Take into account platform-tools version overrides in the specified pipeline",
+)
+def get_platform_helper_for_project(pipeline):
+    """
+    Version precedence is in this order:
+        - if the --pipeline option is supplied, the version in 'platform-config.yml' in:
+            environment_pipelines:
+                <pipeline>:
+                    ...
+                    versions:
+                        platform-helper
+        - The version from default_versions/platform-helper in 'platform-config.yml'
+        - Fall back on the version in the deprecated '.platform-helper-version' file
+    """
+    try:
+        RequiredVersion().get_required_version(pipeline)
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/domain/config_validator.py

@@ -2,11 +2,16 @@ from typing import Callable
 
 import boto3
 
+from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.opensearch import OpensearchProvider
 from dbt_platform_helper.providers.redis import RedisProvider
 
 
+class ConfigValidatorError(PlatformException):
+    pass
+
+
 class ConfigValidator:
 
     def __init__(
@@ -110,7 +115,7 @@ class ConfigValidator:
                 envs = detail["bad_envs"]
                 acc = detail["account"]
                 message += f"  '{pipeline}' - these environments are not in the '{acc}' account: {', '.join(envs)}\n"
-            self.io.abort_with_error(message)
+            raise ConfigValidatorError(message)
 
     def validate_environment_pipelines_triggers(self, config):
         errors = []
@@ -134,7 +139,7 @@ class ConfigValidator:
 
         if errors:
             error_message = "The following pipelines are misconfigured: \n"
-            self.io.abort_with_error(error_message + "\n  ".join(errors))
+            raise ConfigValidatorError(error_message + "\n  ".join(errors))
 
     def validate_database_copy_section(self, config):
         extensions = config.get("extensions", {})
@@ -220,7 +225,7 @@ class ConfigValidator:
                         )
 
         if errors:
-            self.io.abort_with_error("\n".join(errors))
+            raise ConfigValidatorError("\n".join(errors))
 
     def validate_database_migration_input_sources(self, config: dict):
         extensions = config.get("extensions", {})
@@ -247,7 +252,7 @@ class ConfigValidator:
                     )
                 if "import" not in data_migration and "import_sources" not in data_migration:
                     errors.append(
-                        f"Error in '{extension_name}.environments.{env}.data_migration': 'import_sources' property is missing."
+                        f"'import_sources' property in '{extension_name}.environments.{env}.data_migration' is missing."
                     )
         if errors:
-            self.io.abort_with_error("\n".join(errors))
+            raise ConfigValidatorError("\n".join(errors))

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/domain/copilot_environment.py

@@ -1,8 +1,6 @@
 from collections import defaultdict
 from pathlib import Path
-from typing import Callable
 
-import click
 from boto3 import Session
 
 from dbt_platform_helper.domain.terraform_environment import (
@@ -11,6 +9,7 @@ from dbt_platform_helper.domain.terraform_environment import (
 from dbt_platform_helper.providers.cloudformation import CloudFormation
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.load_balancers import (
     get_https_certificate_for_application,
 )
@@ -30,14 +29,14 @@ class CopilotEnvironment:
         cloudformation_provider: CloudFormation = None,
         session: Session = None,  # TODO - this is a temporary fix, will fall away once the Loadbalancer provider is in place.
         copilot_templating=None,
-        echo: Callable[[str], str] = click.secho,
+        io: ClickIOProvider = ClickIOProvider(),
     ):
         self.config_provider = config_provider
         self.vpc_provider = vpc_provider
         self.copilot_templating = copilot_templating or CopilotTemplating(
             file_provider=FileProvider(),
         )
-        self.echo = echo
+        self.io = io
         self.session = session
         self.cloudformation_provider = cloudformation_provider
 
@@ -53,7 +52,7 @@ class CopilotEnvironment:
         env_config = platform_config["environments"][environment_name]
         profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
 
-        self.echo(f"Using {profile_for_environment} for this AWS session")
+        self.io.info(f"Using {profile_for_environment} for this AWS session")
 
         app_name = platform_config["application"]
 
@@ -72,7 +71,7 @@ class CopilotEnvironment:
             cert_arn=certificate_arn,
         )
 
-        self.echo(
+        self.io.info(
             self.copilot_templating.write_environment_manifest(
                 environment_name, copilot_environment_manifest
             )
@@ -134,11 +133,13 @@ class CopilotTemplating:
     def __init__(
         self,
         file_provider: FileProvider = None,
+        io: ClickIOProvider = ClickIOProvider(),
         # TODO file_provider can be moved up a layer.  File writing can be the responsibility of CopilotEnvironment generate
         # Or we align with PlatformTerraformManifestGenerator and rename from Templating to reflect the file writing responsibility
     ):
         self.file_provider = file_provider
         self.templates = setup_templates()
+        self.io = io
 
     def generate_copilot_environment_manifest(
         self, environment_name: str, vpc: Vpc, cert_arn: str
@@ -193,16 +194,16 @@ class CopilotTemplating:
                             )
 
         if not resource_blocks:
-            click.echo("\n>>> No cross-environment S3 policies to create.\n")
+            self.io.info("\n>>> No cross-environment S3 policies to create.\n")
             return
 
         for service in sorted(resource_blocks.keys()):
             resources = resource_blocks[service]
-            click.echo(f"\n>>> Creating S3 cross account policies for {service}.\n")
+            self.io.info(f"\n>>> Creating S3 cross account policies for {service}.\n")
             template = self.templates.get_template(S3_CROSS_ACCOUNT_POLICY)
             file_content = template.render({"resources": resources})
             output_dir = Path(".").absolute()
             file_path = f"copilot/{service}/addons/s3-cross-account-policy.yml"
 
             self.file_provider.mkfile(output_dir, file_path, file_content, True)
-            click.echo(f"File {file_path} created")
+            self.io.info(f"File {file_path} created")

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/domain/database_copy.py

@@ -79,6 +79,9 @@ class DatabaseCopy:
         except VpcProviderException as ex:
             self.io.abort_with_error(str(ex))
 
+        if not vpc_config.security_groups:
+            self.io.abort_with_error(f"No security groups found in vpc '{vpc_name}'")
+
         database_identifier = f"{self.app}-{env}-{self.database}"
 
         try:
@@ -121,7 +124,7 @@ class DatabaseCopy:
         self,
         session: boto3.session.Session,
         env: str,
-        vpc_config: Vpc,
+        vpc: Vpc,
         is_dump: bool,
         db_connection_string: str,
         filename: str,
@@ -145,8 +148,8 @@ class DatabaseCopy:
             ],
             networkConfiguration={
                 "awsvpcConfiguration": {
-                    "subnets": vpc_config.private_subnets,
-                    "securityGroups": vpc_config.security_groups,
+                    "subnets": vpc.private_subnets,
+                    "securityGroups": vpc.security_groups,
                     "assignPublicIp": "DISABLED",
                 }
             },

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/domain/maintenance_page.py

@@ -90,7 +90,7 @@ def add_maintenance_page(
     app: str,
     env: str,
     services: List[Service],
-    allowed_ips: tuple,
+    allowed_ips: List[str],
     template: str = "default",
 ):
     lb_client = session.client("elbv2")
@@ -98,6 +98,7 @@ def add_maintenance_page(
     bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
 
     rule_priority = itertools.count(start=1)
+    maintenance_page_host_header_conditions = []
     try:
         for svc in services:
             target_group_arn = find_target_group(app, env, svc.name, session)
@@ -106,6 +107,10 @@ def add_maintenance_page(
             if not target_group_arn:
                 continue
 
+            service_conditions = get_host_header_conditions(
+                lb_client, listener_arn, target_group_arn
+            )
+
             for ip in allowed_ips:
                 create_header_rule(
                     lb_client,
@@ -115,6 +120,7 @@ def add_maintenance_page(
                     [ip],
                     "AllowedIps",
                     next(rule_priority),
+                    service_conditions,
                 )
                 create_source_ip_rule(
                     lb_client,
@@ -123,6 +129,7 @@ def add_maintenance_page(
                     [ip],
                     "AllowedSourceIps",
                     next(rule_priority),
+                    service_conditions,
                 )
 
             create_header_rule(
@@ -133,8 +140,12 @@ def add_maintenance_page(
                 [bypass_value],
                 "BypassIpFilter",
                 next(rule_priority),
+                service_conditions,
             )
 
+            # add to accumilating list of conditions for maintenace page rule
+            maintenance_page_host_header_conditions.extend(service_conditions)
+
         click.secho(
             f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
         )
@@ -146,7 +157,21 @@ def add_maintenance_page(
                 {
                     "Field": "path-pattern",
                     "PathPatternConfig": {"Values": ["/*"]},
-                }
+                },
+                {
+                    "Field": "host-header",
+                    "HostHeaderConfig": {
+                        "Values": sorted(
+                            list(
+                                {
+                                    value
+                                    for condition in maintenance_page_host_header_conditions
+                                    for value in condition["HostHeaderConfig"]["Values"]
+                                }
+                            )
+                        )
+                    },
+                },
             ],
             Actions=[
                 {
@@ -445,9 +470,8 @@ def create_header_rule(
     values: list,
     rule_name: str,
     priority: int,
+    conditions: list,
 ):
-    conditions = get_host_conditions(lb_client, listener_arn, target_group_arn)
-
     # add new condition to existing conditions
     combined_conditions = [
         {
@@ -486,9 +510,8 @@ def create_source_ip_rule(
     values: list,
     rule_name: str,
     priority: int,
+    conditions: list,
 ):
-    conditions = get_host_conditions(lb_client, listener_arn, target_group_arn)
-
     # add new condition to existing conditions
 
     combined_conditions = [
@@ -514,7 +537,9 @@ def create_source_ip_rule(
     )
 
 
-def get_host_conditions(lb_client: boto3.client, listener_arn: str, target_group_arn: str):
+def get_host_header_conditions(
+    lb_client: boto3.client, listener_arn: str, target_group_arn: str
+) -> list:
     rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
 
     # Get current set of forwarding conditions for the target group

dbt_platform_helper-13.0.2/dbt_platform_helper/domain/terraform_environment.py

@@ -0,0 +1,46 @@
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
+from dbt_platform_helper.utils.versioning import (
+    get_required_terraform_platform_modules_version,
+)
+
+
+class TerraformEnvironmentException(PlatformException):
+    pass
+
+
+class EnvironmentNotFoundException(TerraformEnvironmentException):
+    pass
+
+
+class TerraformEnvironment:
+    def __init__(
+        self,
+        config_provider,
+        manifest_provider: TerraformManifestProvider = None,
+        io: ClickIOProvider = ClickIOProvider(),
+    ):
+        self.io = io
+        self.config_provider = config_provider
+        self.manifest_provider = manifest_provider or TerraformManifestProvider()
+
+    def generate(self, environment_name, terraform_platform_modules_version_override=None):
+        config = self.config_provider.get_enriched_config()
+
+        if environment_name not in config.get("environments").keys():
+            raise EnvironmentNotFoundException(
+                f"cannot generate terraform for environment {environment_name}.  It does not exist in your configuration"
+            )
+
+        platform_config_terraform_modules_default_version = config.get("default_versions", {}).get(
+            "terraform-platform-modules", ""
+        )
+        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
+            terraform_platform_modules_version_override,
+            platform_config_terraform_modules_default_version,
+        )
+
+        self.manifest_provider.generate_environment_config(
+            config, environment_name, terraform_platform_modules_version
+        )

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/providers/config.py

@@ -5,6 +5,7 @@ from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.config_validator import ConfigValidator
+from dbt_platform_helper.domain.config_validator import ConfigValidatorError
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.platform_config_schema import PlatformConfigSchema
 from dbt_platform_helper.providers.yaml_file import FileNotFoundException
@@ -35,7 +36,10 @@ class ConfigProvider:
         # also, we apply defaults but discard that data.  Should we just apply
         # defaults to config returned by load_and_validate
         enriched_config = ConfigProvider.apply_environment_defaults(self.config)
-        self.validator.run_validations(enriched_config)
+        try:
+            self.validator.run_validations(enriched_config)
+        except ConfigValidatorError as exc:
+            self.io.abort_with_error(f"Config validation has failed.\n{str(exc)}")
 
     def load_and_validate_platform_config(self, path=PLATFORM_CONFIG_FILE):
         try:
@@ -54,6 +58,12 @@ class ConfigProvider:
 
         return self.config
 
+    def load_unvalidated_config_file(self, path=PLATFORM_CONFIG_FILE):
+        try:
+            return self.file_provider.load(path)
+        except FileProviderException:
+            return {}
+
     # TODO this general function should be moved out of ConfigProvider
     def config_file_check(self, path=PLATFORM_CONFIG_FILE):
         if not Path(path).exists():

dbt_platform_helper-13.0.2/dbt_platform_helper/providers/files.py

@@ -0,0 +1,27 @@
+from pathlib import Path
+
+
+class FileProvider:
+
+    def load(path: str) -> str:
+        pass
+
+    @staticmethod
+    def mkfile(base_path: str, file_name: str, contents: str, overwrite=False) -> str:
+        file_path = Path(base_path).joinpath(file_name)
+        file_exists = file_path.exists()
+        if file_exists and not overwrite:
+            return f"File {file_path} exists; doing nothing"
+
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        file_path.write_text(contents)
+
+        action = "overwritten" if file_exists and overwrite else "created"
+        return f"File {file_name} {action}"
+
+    @staticmethod
+    def delete_file(base_path: str, file_name: str):
+        file_path = Path(base_path) / file_name
+        if file_path.exists():
+            file_path.unlink()
+            return f"{str(file_path)} has been deleted"

{dbt_platform_helper-13.0.0 → dbt_platform_helper-13.0.2}/dbt_platform_helper/providers/load_balancers.py

@@ -21,7 +21,9 @@ def get_load_balancer_for_application(session: boto3.Session, app: str, env: str
             load_balancer_arn = lb["ResourceArn"]
 
     if not load_balancer_arn:
-        raise LoadBalancerNotFoundException()
+        raise LoadBalancerNotFoundException(
+            f"No load balancer found for {app} in the {env} environment"
+        )
 
     return load_balancer_arn
 
@@ -39,7 +41,7 @@ def get_https_listener_for_application(session: boto3.Session, app: str, env: st
         pass
 
     if not listener_arn:
-        raise ListenerNotFoundException()
+        raise ListenerNotFoundException(f"No HTTPS listener for {app} in the {env} environment")
 
     return listener_arn