dbt-platform-helper 13.1.2__py3-none-any.whl → 13.3.0__py3-none-any.whl

dbt_platform_helper/domain/copilot.py

@@ -5,18 +5,19 @@ import json
 from pathlib import Path
 from pathlib import PosixPath
 
-import click
-import yaml
-from schema import SchemaError
+import botocore
+import botocore.errorfactory
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.kms import KMSProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import get_aws_session_or_abort
-from dbt_platform_helper.utils.files import generate_override_files
 from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
@@ -27,9 +28,6 @@ class Copilot:
 
     PACKAGE_DIR = Path(__file__).resolve().parent.parent
 
-    # TODO Remove and test
-    WAF_ACL_ARN_KEY = "waf-acl-arn"
-
     SERVICE_TYPES = [
         "Load Balanced Web Service",
         "Backend Service",
@@ -41,34 +39,115 @@ class Copilot:
     def __init__(
         self,
         config_provider: ConfigProvider,
+        parameter_provider: ParameterStore,
         file_provider: FileProvider,
         copilot_templating: CopilotTemplating,
+        kms_provider: KMSProvider,
+        session,
+        io: ClickIOProvider = ClickIOProvider(),
+        yaml_file_provider: YamlFileProvider = YamlFileProvider,
     ):
         self.config_provider = config_provider
+        self.parameter_provider = parameter_provider
         self.file_provider = file_provider
         self.copilot_templating = copilot_templating
+        self.kms_provider = kms_provider
+        self.io = io
+        self.yaml_file_provider = yaml_file_provider
+        self.session = session
+
+    def make_addons(self):
+        config = self.config_provider.load_and_validate_platform_config()
+
+        templates = setup_templates()
+        extensions = self._get_extensions()
+        application_name = get_application_name()
+
+        self.io.info("\n>>> Generating Terraform compatible addons CloudFormation\n")
+
+        output_dir = Path(".").absolute()
+        env_path = Path(f"copilot/environments/")
+        env_addons_path = env_path / "addons"
+        env_overrides_path = env_path / "overrides"
+
+        self._cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
+        self._generate_env_overrides(output_dir)
+
+        svc_names = self._list_copilot_local_services()
+        base_path = Path(".")
+        for svc_name in svc_names:
+            self._generate_svc_overrides(base_path, templates, svc_name)
 
-    def list_copilot_local_environments(self):
+        services = []
+        for ext_name, ext_data in extensions.items():
+            extension = {**ext_data}
+            addon_type = extension.pop("type")
+            environments = extension.pop("environments")
+            environment_addon_config = {
+                "addon_type": addon_type,
+                "environments": environments,
+                "name": extension.get("name", None) or ext_name,
+                "prefix": camel_case(ext_name),
+                "secret_name": ext_name.upper().replace("-", "_"),
+                **extension,
+            }
+
+            services.append(environment_addon_config)
+
+            service_addon_config = {
+                "application_name": application_name,
+                "name": extension.get("name", None) or ext_name,
+                "prefix": camel_case(ext_name),
+                "environments": environments,
+                **extension,
+            }
+
+            log_destination_arns = self._get_log_destination_arn()
+
+            if addon_type in ["s3", "s3-policy"]:
+                if extensions[ext_name].get("serve_static_content"):
+                    continue
+
+                s3_kms_arns = self._get_s3_kms_alias_arns(application_name, environments)
+                for environment_name in environments:
+                    environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
+                        environment_name, "kms-key-not-found"
+                    )
+
+            self._generate_service_addons(
+                extension,
+                ext_name,
+                addon_type,
+                output_dir,
+                service_addon_config,
+                templates,
+                log_destination_arns,
+            )
+
+        environments = self.config_provider.apply_environment_defaults(config)["environments"]
+
+        self.copilot_templating.generate_cross_account_s3_policies(environments, extensions)
+
+        self.io.info(templates.get_template("addon-instructions.txt").render(services=services))
+
+    def _list_copilot_local_environments(self):
         return [
             path.parent.parts[-1] for path in Path("./copilot/environments/").glob("*/manifest.yml")
         ]
 
-    def is_service(self, path: PosixPath) -> bool:
-        with open(path) as manifest_file:
-            data = yaml.safe_load(manifest_file)
-            if not data or not data.get("type"):
-                click.echo(
-                    click.style(f"No type defined in manifest file {str(path)}; exiting", fg="red")
-                )
-                exit(1)
+    def _is_service(self, path: PosixPath) -> bool:
+
+        manifest_file = self.yaml_file_provider.load(path)
+        if not manifest_file or not manifest_file.get("type"):
+            self.io.abort_with_error(f"No type defined in manifest file {str(path)}; exiting")
 
-            return data.get("type") in self.SERVICE_TYPES
+        return manifest_file.get("type") in self.SERVICE_TYPES
 
-    def list_copilot_local_services(self):
+    def _list_copilot_local_services(self):
         return [
             path.parent.parts[-1]
             for path in Path("./copilot/").glob("*/manifest.yml")
-            if self.is_service(path)
+            if self._is_service(path)
         ]
 
     def _validate_and_normalise_extensions_config(self, config_file, key_in_config_file=None):
@@ -90,12 +169,10 @@ class Copilot:
         def _normalise_keys(source: dict):
             return {k.replace("-", "_"): v for k, v in source.items()}
 
-        with open(self.PACKAGE_DIR / "addon-plans.yml", "r") as fd:
-            addon_plans = yaml.safe_load(fd)
+        addon_plans = self.yaml_file_provider.load(self.PACKAGE_DIR / "addon-plans.yml")
 
         # load and validate config
-        with open(config_file, "r") as fd:
-            config = yaml.safe_load(fd)
+        config = self.yaml_file_provider.load(config_file)
 
         if config and key_in_config_file:
             config = config[key_in_config_file]
@@ -107,23 +184,19 @@ class Copilot:
         errors = validate_addons(config)
 
         if errors:
-            click.echo(click.style(f"Errors found in {config_file}:", fg="red"))
+            self.io.error(f"Errors found in {config_file}:")
             for addon, error in errors.items():
-                click.echo(click.style(f"Addon '{addon}': {error}", fg="red"))
-            exit(1)
+                self.io.error(f"Addon '{addon}': {error}")
+            self.io.abort_with_error("Invalid platform-config.yml provided, see above warnings")
 
-        env_names = self.list_copilot_local_environments()
-        svc_names = self.list_copilot_local_services()
+        env_names = self._list_copilot_local_environments()
+        svc_names = self._list_copilot_local_services()
 
         if not env_names:
-            click.echo(
-                click.style(f"No environments found in ./copilot/environments; exiting", fg="red")
-            )
-            exit(1)
+            self.io.abort_with_error("No environments found in ./copilot/environments; exiting")
 
         if not svc_names:
-            click.echo(click.style(f"No services found in ./copilot/; exiting", fg="red"))
-            exit(1)
+            self.io.abort_with_error("No services found in ./copilot/; exiting")
 
         normalised_config = {}
         config_has_errors = False
@@ -136,11 +209,8 @@ class Copilot:
                     normalised_config[addon_name]["services"] = svc_names
 
                 if not set(normalised_config[addon_name]["services"]).issubset(set(svc_names)):
-                    click.echo(
-                        click.style(
-                            f"Services listed in {addon_name}.services do not exist in ./copilot/",
-                            fg="red",
-                        ),
+                    self.io.error(
+                        f"Services listed in {addon_name}.services do not exist in ./copilot/"
                     )
                     config_has_errors = True
 
@@ -151,18 +221,10 @@ class Copilot:
 
             missing_envs = set(environments.keys()) - set(env_names)
             if missing_envs:
-                click.echo(
-                    click.style(
-                        f"Environment keys listed in {addon_name} do not match those defined in ./copilot/environments.",
-                        fg="red",
-                    )
-                ),
-                click.echo(
-                    click.style(
-                        f"  Missing environments: {', '.join(sorted(missing_envs))}",
-                        fg="white",
-                    ),
+                self.io.error(
+                    f"Environment keys listed in {addon_name} do not match those defined in ./copilot/environments"
                 )
+                self.io.error(f"  Missing environments: {', '.join(sorted(missing_envs))}")
                 config_has_errors = True
 
             if config_has_errors:
@@ -183,41 +245,41 @@ class Copilot:
             normalised_config[addon_name]["environments"] = normalised_environments
 
         if config_has_errors:
-            exit(1)
+            self.io.abort_with_error("Configuration has errors. Exiting.")
 
         return normalised_config
 
-    def get_log_destination_arn(self):
+    def _get_log_destination_arn(self):
         """Get destination arns stored in param store in projects aws
         account."""
-        session = get_aws_session_or_abort()
-        client = session.client("ssm", region_name="eu-west-2")
-        response = client.get_parameters(Names=["/copilot/tools/central_log_groups"])
-
-        if not response["Parameters"]:
-            click.echo(
-                click.style(
-                    "No aws central log group defined in Parameter Store at location /copilot/tools/central_log_groups; exiting",
-                    fg="red",
-                )
+
+        try:
+            destination_arns = self.parameter_provider.get_ssm_parameter_by_name(
+                "/copilot/tools/central_log_groups"
+            )
+        except botocore.errorfactory.ParameterNotFound:
+            self.io.abort_with_error(
+                "No aws central log group defined in Parameter Store at location /copilot/tools/central_log_groups; exiting"
             )
-            exit(1)
 
-        destination_arns = json.loads(response["Parameters"][0]["Value"])
-        return destination_arns
+        return json.loads(destination_arns["Value"])
 
     def _generate_svc_overrides(self, base_path, templates, name):
-        click.echo(f"\n>>> Generating service overrides for {name}\n")
+        self.io.info(f"\n>>> Generating service overrides for {name}\n")
         overrides_path = base_path.joinpath(f"copilot/{name}/overrides")
         overrides_path.mkdir(parents=True, exist_ok=True)
         overrides_file = overrides_path.joinpath("cfn.patches.yml")
         overrides_file.write_text(templates.get_template("svc/overrides/cfn.patches.yml").render())
 
-    def _get_s3_kms_alias_arns(self, session, application_name, config):
-        application = load_application(application_name, session)
+    def _get_s3_kms_alias_arns(self, application_name, config):
+        application = load_application(application_name, self.session)
         arns = {}
 
         for environment_name in application.environments:
+            kms_provider = self.kms_provider(
+                application.environments[environment_name].session.client("kms")
+            )
+
             if environment_name not in config:
                 continue
 
@@ -225,99 +287,20 @@ class Copilot:
                 continue
 
             bucket_name = config[environment_name]["bucket_name"]
-            kms_client = application.environments[environment_name].session.client("kms")
             alias_name = f"alias/{application_name}-{environment_name}-{bucket_name}-key"
 
             try:
-                response = kms_client.describe_key(KeyId=alias_name)
-            except kms_client.exceptions.NotFoundException:
-                pass
+                response = kms_provider.describe_key(alias_name)
+
+            # Boto3 classifies all AWS service errors and exceptions as ClientError exceptions
+            except botocore.exceptions.ClientError as error:
+                if error.response["Error"]["Code"] == "NotFoundException":
+                    pass
             else:
                 arns[environment_name] = response["KeyMetadata"]["Arn"]
 
         return arns
 
-    def make_addons(self):
-        self.config_provider.config_file_check()
-        try:
-            config = self.config_provider.load_and_validate_platform_config()
-        except SchemaError as ex:
-            click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
-            raise click.Abort
-
-        templates = setup_templates()
-        extensions = self._get_extensions()
-        session = get_aws_session_or_abort()
-
-        application_name = get_application_name()
-
-        click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
-
-        output_dir = Path(".").absolute()
-        env_path = Path(f"copilot/environments/")
-        env_addons_path = env_path / "addons"
-        env_overrides_path = env_path / "overrides"
-
-        self._cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
-        self._generate_env_overrides(output_dir)
-
-        svc_names = self.list_copilot_local_services()
-        base_path = Path(".")
-        for svc_name in svc_names:
-            self._generate_svc_overrides(base_path, templates, svc_name)
-
-        services = []
-        for ext_name, ext_data in extensions.items():
-            extension = {**ext_data}
-            addon_type = extension.pop("type")
-            environments = extension.pop("environments")
-            environment_addon_config = {
-                "addon_type": addon_type,
-                "environments": environments,
-                "name": extension.get("name", None) or ext_name,
-                "prefix": camel_case(ext_name),
-                "secret_name": ext_name.upper().replace("-", "_"),
-                **extension,
-            }
-
-            services.append(environment_addon_config)
-
-            service_addon_config = {
-                "application_name": application_name,
-                "name": extension.get("name", None) or ext_name,
-                "prefix": camel_case(ext_name),
-                "environments": environments,
-                **extension,
-            }
-
-            log_destination_arns = self.get_log_destination_arn()
-
-            if addon_type in ["s3", "s3-policy"]:
-                if extensions[ext_name].get("serve_static_content"):
-                    continue
-
-                s3_kms_arns = self._get_s3_kms_alias_arns(session, application_name, environments)
-                for environment_name in environments:
-                    environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
-                        environment_name, "kms-key-not-found"
-                    )
-
-            self._generate_service_addons(
-                extension,
-                ext_name,
-                addon_type,
-                output_dir,
-                service_addon_config,
-                templates,
-                log_destination_arns,
-            )
-
-        environments = self.config_provider.apply_environment_defaults(config)["environments"]
-
-        self.copilot_templating.generate_cross_account_s3_policies(environments, extensions)
-
-        click.echo(templates.get_template("addon-instructions.txt").render(services=services))
-
     def _get_extensions(self):
         config = self._validate_and_normalise_extensions_config(
             self.PACKAGE_DIR / "default-extensions.yml"
@@ -328,13 +311,31 @@ class Copilot:
         config.update(project_config)
         return config
 
+    def _generate_override_files(self, base_path, file_path, output_dir):
+        def generate_files_for_dir(pattern):
+            for file in file_path.glob(pattern):
+                if file.is_file():
+                    contents = file.read_text()
+                    file_name = str(file).removeprefix(f"{file_path}/")
+                    self.io.info(
+                        self.file_provider.mkfile(
+                            base_path,
+                            output_dir / file_name,
+                            contents,
+                            overwrite=True,
+                        )
+                    )
+
+        generate_files_for_dir("*")
+        generate_files_for_dir("bin/*")
+
     def _generate_env_overrides(self, output_dir):
         path = "templates/env/terraform-overrides"
-        click.echo("\n>>> Generating Environment overrides\n")
+        self.io.info("\n>>> Generating Environment overrides\n")
         overrides_path = output_dir.joinpath(f"copilot/environments/overrides")
         overrides_path.mkdir(parents=True, exist_ok=True)
         template_overrides_path = Path(__file__).parent.parent.joinpath(path)
-        generate_override_files(Path("."), template_overrides_path, overrides_path)
+        self._generate_override_files(Path("."), template_overrides_path, overrides_path)
 
     def _generate_service_addons(
         self,
@@ -361,7 +362,7 @@ class Copilot:
                 )
 
                 (output_dir / service_path).mkdir(parents=True, exist_ok=True)
-                click.echo(
+                self.io.info(
                     self.file_provider.mkfile(
                         output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
                     )

dbt_platform_helper/domain/maintenance_page.py

@@ -38,20 +38,14 @@ class FailedToActivateMaintenancePageException(MaintenancePageException):
         application_name: str,
         env: str,
         original_exception: Exception,
-        rolled_back_rules: dict[str, bool] = {},
     ):
         super().__init__(
             f"Maintenance page failed to activate for the {application_name} application in environment {env}."
         )
         self.orginal_exception = original_exception
-        self.rolled_back_rules = rolled_back_rules
 
     def __str__(self):
-        return (
-            f"{super().__str__()}\n"
-            f"Rolled-back rules: {self.rolled_back_rules }\n"
-            f"Original exception: {self.orginal_exception}"
-        )
+        return f"{super().__str__()}\n" f"Original exception: {self.orginal_exception}"
 
 
 # TODO should this be in its own provider, inside the VPC one, what logic is this sepcific too?
@@ -187,6 +181,15 @@ class MaintenancePage:
                     listener_arn, target_group_arn
                 )
 
+                self.io.debug(
+                    f"""
+#----------------------------------------------------------#
+# Creating listener rules for service {svc.name.ljust(21, " ")}#
+#----------------------------------------------------------#
+
+""",
+                )
+
                 for ip in allowed_ips:
                     self.load_balancer.create_header_rule(
                         listener_arn,
@@ -196,6 +199,7 @@ class MaintenancePage:
                         "AllowedIps",
                         next(rule_priority),
                         service_conditions,
+                        [{"Key": "service", "Value": svc.name}],
                     )
                     self.load_balancer.create_source_ip_rule(
                         listener_arn,
@@ -204,6 +208,7 @@ class MaintenancePage:
                         "AllowedSourceIps",
                         next(rule_priority),
                         service_conditions,
+                        [{"Key": "service", "Value": svc.name}],
                     )
 
                 self.load_balancer.create_header_rule(
@@ -214,6 +219,7 @@ class MaintenancePage:
                     "BypassIpFilter",
                     next(rule_priority),
                     service_conditions,
+                    [{"Key": "service", "Value": svc.name}],
                 )
 
                 # add to accumilating list of conditions for maintenace page rule
@@ -262,27 +268,48 @@ class MaintenancePage:
                 ],
             )
         except Exception as e:
-            deleted_rules = self.__clean_up_maintenance_page_rules(listener_arn)
+            self.__clean_up_maintenance_page_rules(listener_arn)
             raise FailedToActivateMaintenancePageException(
-                app, env, f"{e}:\n {traceback.format_exc()}", deleted_rules
+                app, env, f"{e}:\n {traceback.format_exc()}"
             )
 
     def __clean_up_maintenance_page_rules(
         self, listener_arn: str, fail_when_not_deleted: bool = False
-    ) -> dict[str, bool]:
+    ) -> None:
 
         tag_descriptions = self.load_balancer.get_rules_tag_descriptions_by_listener_arn(
             listener_arn
         )
 
-        deletes = {}
+        # keep track of rules deleted
+        deleted_rules = {"MaintenancePage": 0}
         for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
-            deleted = self.load_balancer.delete_listener_rule_by_tags(tag_descriptions, name)
-            deletes[name] = bool(deleted)
-            if fail_when_not_deleted and name == "MaintenancePage" and not deleted:
+            deleted_list = self.load_balancer.delete_listener_rule_by_tags(tag_descriptions, name)
+
+            # track the rules deleted grouped by service
+            for deleted_rule in deleted_list:
+                tags = {t["Key"]: t["Value"] for t in deleted_rule["Tags"]}
+                if "service" in tags:
+                    if tags["service"] not in deleted_rules:
+                        deleted_rules[tags["service"]] = {
+                            "AllowedIps": 0,
+                            "BypassIpFilter": 0,
+                            "AllowedSourceIps": 0,
+                        }
+                    deleted_rules[tags["service"]][name] += 1
+                elif tags.get("name") == "MaintenancePage":
+                    deleted_rules["MaintenancePage"] += 1
+
+            if (
+                fail_when_not_deleted
+                and name == "MaintenancePage"
+                and deleted_rules["MaintenancePage"] == 0
+            ):
                 raise ListenerRuleNotFoundException()
 
-        return deletes
+        self.io.warn(
+            f"Rules deleted by type and grouped by service: {deleted_rules}",
+        )
 
     def __remove_maintenance_page(self, listener_arn: str) -> dict[str, bool]:
         self.__clean_up_maintenance_page_rules(listener_arn, True)

dbt_platform_helper/domain/pipelines.py

@@ -14,7 +14,7 @@ from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
 from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.versioning import (
+from dbt_platform_helper.utils.tool_versioning import (
     get_required_terraform_platform_modules_version,
 )
 

dbt_platform_helper/domain/terraform_environment.py

@@ -1,7 +1,7 @@
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
-from dbt_platform_helper.utils.versioning import (
+from dbt_platform_helper.utils.tool_versioning import (
     get_required_terraform_platform_modules_version,
 )