dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/domain/notify.py

@@ -0,0 +1,64 @@
+from dbt_platform_helper.providers.slack_channel_notifier import SlackChannelNotifier
+from dbt_platform_helper.utils.arn_parser import ARN
+
+
+class Notify:
+    def __init__(self, notifier: SlackChannelNotifier):
+        self.notifier = notifier
+
+    def post_message(
+        self,
+        message: str,
+        build_arn: str = None,
+        repository: str = None,
+        commit_sha: str = None,
+        original_message_ref: str = None,
+    ):
+        context = []
+
+        if repository:
+            context.append(f"*Repository*: <https://github.com/{repository}|{repository}>")
+            if commit_sha:
+                context.append(
+                    f"*Revision*: <https://github.com/{repository}/commit/{commit_sha}|{commit_sha}>"
+                )
+
+        if build_arn:
+            context.append(f"<{get_build_url(build_arn)}|Build Logs>")
+
+        if original_message_ref:
+            return self.notifier.post_update(original_message_ref, message, context)
+        else:
+            return self.notifier.post_new(message, context)
+
+    def add_comment(
+        self,
+        original_message_ref: str,
+        message: str,
+        title: str,
+        reply_broadcast: bool,
+    ):
+        self.notifier.post_new(
+            message=message,
+            title=title,
+            context=[],
+            reply_broadcast=reply_broadcast,
+            thread_ref=original_message_ref,
+        )
+
+
+def get_build_url(build_arn: str):
+    try:
+        arn = ARN(build_arn)
+        url = (
+            "https://{region}.console.aws.amazon.com/codesuite/codebuild/{account}/projects/{"
+            "project}/build/{project}%3A{build_id}"
+        )
+        return url.format(
+            region=arn.region,
+            account=arn.account_id,
+            project=arn.project.replace("build/", ""),
+            build_id=arn.build_id,
+        )
+    except ValueError:
+        return ""

dbt_platform_helper/domain/pipelines.py

@@ -7,16 +7,13 @@ from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
 from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
 from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
+from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.ecr import ECRProvider
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
-from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.template import setup_templates
-from dbt_platform_helper.utils.versioning import (
-    get_required_terraform_platform_modules_version,
-)
 
 
 class Pipelines:
@@ -26,19 +23,43 @@ class Pipelines:
         terraform_manifest_provider: TerraformManifestProvider,
         ecr_provider: ECRProvider,
         get_git_remote: Callable[[], str],
-        get_codestar_arn: Callable[[str], str],
         io: ClickIOProvider = ClickIOProvider(),
         file_provider: FileProvider = FileProvider(),
+        platform_helper_versioning: PlatformHelperVersioning = None,
     ):
         self.config_provider = config_provider
         self.get_git_remote = get_git_remote
-        self.get_codestar_arn = get_codestar_arn
         self.terraform_manifest_provider = terraform_manifest_provider
         self.ecr_provider = ecr_provider
         self.io = io
         self.file_provider = file_provider
+        self.platform_helper_versioning = platform_helper_versioning
+
+    def _map_environment_pipeline_accounts(self, platform_config) -> list[tuple[str, str]]:
+        environment_pipelines_config = platform_config[ENVIRONMENT_PIPELINES_KEY]
+        environment_config = platform_config["environments"]
+
+        account_id_lookup = {
+            env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
+            for env in environment_config.values()
+            if env is not None and "accounts" in env and "deploy" in env["accounts"]
+        }
+
+        accounts = set()
+
+        for config in environment_pipelines_config.values():
+            account = config.get("account")
+            deploy_account_id = account_id_lookup.get(account)
+            accounts.add((account, deploy_account_id))
+
+        return list(accounts)
+
+    def generate(
+        self,
+        deploy_branch: str,
+    ):
+        self.platform_helper_versioning.check_platform_helper_version_mismatch()
 
-    def generate(self, cli_terraform_platform_modules_version: str, deploy_branch: str):
         platform_config = self.config_provider.load_and_validate_platform_config()
 
         has_codebase_pipelines = CODEBASE_PIPELINES_KEY in platform_config
@@ -48,31 +69,16 @@ class Pipelines:
             self.io.warn("No pipelines defined: nothing to do.")
             return
 
-        platform_config_terraform_modules_default_version = platform_config.get(
-            "default_versions", {}
-        ).get("terraform-platform-modules", "")
-
-        app_name = get_application_name()
-
         git_repo = self.get_git_remote()
         if not git_repo:
             self.io.abort_with_error("The current directory is not a git repository")
 
-        codestar_connection_arn = self.get_codestar_arn(app_name)
-        if codestar_connection_arn is None:
-            self.io.abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
-
         base_path = Path(".")
         copilot_pipelines_dir = base_path / f"copilot/pipelines"
 
         self._clean_pipeline_config(copilot_pipelines_dir)
 
-        terraform_platform_modules_version = get_required_terraform_platform_modules_version(
-            cli_terraform_platform_modules_version,
-            platform_config_terraform_modules_default_version,
-        )
-
-        # TODO - this whole code block/if-statement can fall away once the deploy_repository is a required key.
+        # TODO: DBTP-1965: - this whole code block/if-statement can fall away once the deploy_repository is a required key.
         deploy_repository = ""
         if "deploy_repository" in platform_config.keys():
             deploy_repository = f"{platform_config['deploy_repository']}"
@@ -83,20 +89,17 @@ class Pipelines:
             deploy_repository = f"uktrade/{platform_config['application']}-deploy"
 
         if has_environment_pipelines:
-            environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
-            accounts = {
-                config.get("account")
-                for config in environment_pipelines.values()
-                if "account" in config
-            }
+            accounts = self._map_environment_pipeline_accounts(platform_config)
 
-            for account in accounts:
+            for account_name, account_id in accounts:
                 self._generate_terraform_environment_pipeline_manifest(
                     platform_config["application"],
                     deploy_repository,
-                    account,
-                    terraform_platform_modules_version,
+                    account_name,
+                    self.platform_helper_versioning.get_environment_pipeline_modules_source(),
                     deploy_branch,
+                    account_id,
+                    self.platform_helper_versioning.get_pinned_version(),
                 )
 
         if has_codebase_pipelines:
@@ -114,9 +117,10 @@ class Pipelines:
 
             self.terraform_manifest_provider.generate_codebase_pipeline_config(
                 platform_config,
-                terraform_platform_modules_version,
+                self.platform_helper_versioning.get_template_version(),
                 ecrs_that_need_importing,
                 deploy_repository,
+                self.platform_helper_versioning.get_codebase_pipeline_modules_source(),
             )
 
     def _clean_pipeline_config(self, pipelines_dir: Path):
@@ -129,8 +133,10 @@ class Pipelines:
         application: str,
         deploy_repository: str,
         aws_account: str,
-        terraform_platform_modules_version: str,
+        module_source: str,
         deploy_branch: str,
+        aws_account_id: str,
+        pinned_version: str,
     ):
         env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
 
@@ -139,10 +145,12 @@ class Pipelines:
                 "application": application,
                 "deploy_repository": deploy_repository,
                 "aws_account": aws_account,
-                "terraform_platform_modules_version": terraform_platform_modules_version,
+                "module_source": module_source,
                 "deploy_branch": deploy_branch,
                 "terraform_version": SUPPORTED_TERRAFORM_VERSION,
                 "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,
+                "deploy_account_id": aws_account_id,
+                "pinned_version": pinned_version,
             }
         )
 

dbt_platform_helper/domain/plans.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+class PlanLoader:
+
+    PROJECT_DIR = Path(__file__).resolve().parent.parent.parent
+
+    def __init__(
+        self,
+        extensions: dict = None,
+        terraform_dir: str = "terraform",
+        loader: YamlFileProvider = YamlFileProvider,
+    ):
+        self.path = terraform_dir
+        self.loader = loader
+        self._cache = {}
+        self.extensions = extensions or {
+            "redis": "elasticache-redis",
+            "opensearch": "opensearch",
+            "postgres": "postgres",
+        }
+
+    def load(self):
+        result = {}
+        for key, value in self.extensions.items():
+            result[key] = self._load_plan(key, f"{self.PROJECT_DIR}/{self.path}/{value}/plans.yml")
+        return result
+
+    def _load_plan(self, name, path):
+        if name in self._cache:
+            return self._cache[name]
+        else:
+            plan = self.loader.load(path)
+            self._cache[name] = plan
+            return plan
+
+    def get_plan_names(self, extension):
+        plans = self.load()
+        return list(plans[extension].keys())

dbt_platform_helper/domain/secrets.py

@@ -0,0 +1,279 @@
+import botocore
+
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM_TERRAFORM
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import Parameter
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.utils.application import load_application
+
+
+class Secrets:
+
+    def __init__(
+        self,
+        load_application=load_application,
+        io: ClickIOProvider = ClickIOProvider(),
+        parameter_store_provider: ParameterStore = ParameterStore,
+    ):
+        self.load_application_fn = load_application
+        self.application = None
+        self.io = io
+        self.parameter_store_provider: ParameterStore = parameter_store_provider
+
+    def _check_ssm_write_access(self, accounts):
+        """Check access."""
+        no_access = []
+        for account, session in accounts.items():
+            sts = session.client("sts")
+            iam = session.client("iam")
+
+            sts_arn = sts.get_caller_identity()["Arn"]
+            role_name = sts_arn.split("/")[1]
+
+            role_arn = (
+                f"arn:aws:iam::{account}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}"
+            )
+            response = iam.simulate_principal_policy(
+                PolicySourceArn=role_arn,
+                ActionNames=[
+                    "ssm:PutParameter",
+                ],
+                ContextEntries=[
+                    {
+                        "ContextKeyName": "aws:RequestedRegion",
+                        "ContextKeyValues": [
+                            "eu-west-2",
+                        ],
+                        "ContextKeyType": "string",
+                    }
+                ],
+            )["EvaluationResults"]
+
+            has_access = [
+                account for eval_result in response if eval_result["EvalDecision"] == "allowed"
+            ]
+
+            if not has_access:
+                no_access.append(account)
+
+        if no_access:
+            account_ids = "', '".join(no_access)
+            raise PlatformException(
+                f"You do not have AWS Parameter Store write access to the following AWS accounts: '{account_ids}'"
+            )
+
+    def _check_for_existing_params(self, get_secret_name):
+        found_params = []
+        for _, environment in self.application.environments.items():
+            parameter_store: ParameterStore = self.parameter_store_provider(
+                environment.session.client("ssm")
+            )
+            try:
+                param = parameter_store.get_ssm_parameter_by_name(get_secret_name(environment.name))
+                if param:
+                    found_params.append(environment.name)
+            except botocore.exceptions.ClientError as error:
+                if error.response["Error"]["Code"] == "ParameterNotFound":
+                    pass
+                else:
+                    raise PlatformException(error)
+
+        return found_params
+
+    def create(self, app_name, name, overwrite):
+        self.application = (
+            self.load_application_fn(app_name) if not self.application else self.application
+        )
+
+        accounts = {}
+        for _, environment in self.application.environments.items():
+            if environment.account_id not in accounts:
+                accounts[environment.account_id] = environment.session
+
+        self._check_ssm_write_access(accounts)
+
+        get_secret_name = lambda env: f"/platform/{app_name}/{env}/secrets/{name.upper()}"
+        found_params = self._check_for_existing_params(get_secret_name)
+
+        if overwrite is False and found_params:
+            envs = "', '".join(found_params)
+            raise PlatformException(
+                f"AWS Parameter Store secret '{name.upper()}' already exists for the following environments: '{envs}'. \nUse the --overwrite flag to replacing existing secret values."
+            )
+
+        values = {}
+        for _, environment in self.application.environments.items():
+            value = self.io.input(
+                f"Please enter value for secret '{name.upper()}' in environment '{environment.name}'",
+                hide_input=True,
+            )
+            values[environment.name] = value
+
+        for environment_name, secret_value in values.items():
+
+            environment = self.application.environments[environment_name]
+            parameter_store: ParameterStore = self.parameter_store_provider(
+                environment.session.client("ssm")
+            )
+
+            data_dict = dict(
+                Name=get_secret_name(environment.name),
+                Value=secret_value,
+                Overwrite=overwrite,
+                Type="SecureString",
+                Tags=[
+                    {"Key": "application", "Value": app_name},
+                    {"Key": "environment", "Value": environment.name},
+                    {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
+                ],
+            )
+
+            # If in found params we are overwriting
+            if overwrite and environment_name in found_params:
+                data_dict["Overwrite"] = True
+                del data_dict["Tags"]
+            self.io.debug(
+                f"Creating AWS Parameter Store secret {get_secret_name(environment.name)} ..."
+            )
+            parameter_store.put_parameter(data_dict)
+            self.io.debug(
+                f"Successfully created AWS Parameter Store secret {get_secret_name(environment.name)}"
+            )
+
+        self.io.info(
+            "\nTo check or update your secrets, log into your AWS account via the Console and visit the Parameter Store https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/\n"
+            "You can attach secrets into ECS container by adding them to the `secrets` section of your 'service-config.yml' file."
+        )
+
+        self.io.info(
+            message=f"```\nsecrets:\n\t{name.upper()}: /platform/${{PLATFORM_APPLICATION_NAME}}/${{PLATFORM_ENVIRONMENT_NAME}}/secrets/{name.upper()}\n```",
+            fg="cyan",
+            bold=True,
+        )
+
+    def __has_access(self, env, actions=["ssm:PutParameter"], access_type="write"):
+        sts_arn = env.session.client("sts").get_caller_identity()["Arn"]
+        role_name = sts_arn.split("/")[1]
+
+        role_arn = f"arn:aws:iam::{env.account_id}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}"
+        response = env.session.client("iam").simulate_principal_policy(
+            PolicySourceArn=role_arn,
+            ActionNames=actions,
+            ContextEntries=[
+                {
+                    "ContextKeyName": "aws:RequestedRegion",
+                    "ContextKeyValues": [
+                        "eu-west-2",
+                    ],
+                    "ContextKeyType": "string",
+                }
+            ],
+        )["EvaluationResults"]
+        has_access = [
+            env.account_id for eval_result in response if eval_result["EvalDecision"] == "allowed"
+        ]
+
+        if not has_access:
+            raise PlatformException(
+                f"You do not have AWS Parameter Store {access_type} access to the following AWS accounts: '{env.account_id}'"
+            )
+
+    def copy(self, app_name: str, source: str, target: str):
+        """Copy AWS Parameter Store secrets from one environment into
+        another."""
+
+        self.application = (
+            self.load_application_fn(app_name) if not self.application else self.application
+        )
+
+        if not self.application.environments.get(target, ""):
+            raise PlatformException(
+                f"Environment '{target}' not found for application '{app_name}'."
+            )
+        elif not self.application.environments.get(source, ""):
+            raise PlatformException(
+                f"Environment '{source}' not found for application '{app_name}'."
+            )
+
+        source_env = self.application.environments.get(source)
+        target_env = self.application.environments.get(target)
+
+        self.__has_access(source_env, actions=["ssm:GetParameter"], access_type="read")
+        self.__has_access(target_env)
+
+        prod_account_id = self.application.environments["prod"].account_id
+        if (
+            self.application.environments[source].account_id == prod_account_id
+            and self.application.environments[target].account_id != prod_account_id
+        ):
+            raise PlatformException(
+                f"Cannot transfer secrets out from '{source}' in the prod account '{prod_account_id}'"
+                f" to '{target}' in '{self.application.environments[target].account_id}'"
+            )
+
+        parameter_store: ParameterStore = self.parameter_store_provider(
+            source_env.session.client("ssm"), with_model=True
+        )
+
+        target_parameter_store: ParameterStore = self.parameter_store_provider(
+            target_env.session.client("ssm"), with_model=True
+        )
+
+        copilot_secrets: list[Parameter] = parameter_store.get_ssm_parameters_by_path(
+            f"/copilot/{app_name}/{source}/secrets", add_tags=True
+        )
+        platform_secrets: list[Parameter] = parameter_store.get_ssm_parameters_by_path(
+            f"/platform/{app_name}/{source}/secrets", add_tags=True
+        )
+
+        secrets = copilot_secrets + platform_secrets
+
+        for secret in secrets:
+            secret.name = secret.name.replace(f"/{source}/", f"/{target}/")
+
+            if (
+                "AWS_" in secret.name
+                or secret.tags.get("managed-by", "") == MANAGED_BY_PLATFORM_TERRAFORM
+                or secret.tags.get("managed-by", "")
+                == "Terraform"  # SSM params POSTGRES_APPLICATION_USER and POSTGRES_READ_ONLY_USER are tagged differently from the rest
+            ):
+                message = f"Skipping AWS Parameter Store secret {secret.name}"
+                if secret.tags.get("managed-by", ""):
+                    managed_by = secret.tags["managed-by"]
+                    message += f" with managed-by: {managed_by}"
+                self.io.debug(message)
+                continue
+
+            secret.tags["application"] = app_name
+            secret.tags["environment"] = target
+            secret.tags["managed-by"] = MANAGED_BY_PLATFORM
+            secret.tags["copied-from"] = source
+
+            if secret.name.startswith("/copilot/"):
+                secret.tags["copilot-environment"] = target
+
+            data_dict = dict(
+                Name=secret.name,
+                Value=secret.value,
+                Overwrite=False,
+                Type=secret.type,
+                Description=f"Copied from {source} environment.",
+                Tags=secret.tags_to_list(),
+            )
+            self.io.debug(f"Creating AWS Parameter Store secret {secret.name} ...")
+
+            try:
+                target_parameter_store.put_parameter(data_dict)
+                secret_name = secret.name.split("/")[-1]
+                self.io.info(
+                    f"Secret {secret_name} was successfully copied from the '{source} environment to '{target}'"
+                )
+            except botocore.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] == "ParameterAlreadyExists":
+                    self.io.warn(
+                        f"""The "{secret.name.split("/")[-1]}" parameter already exists for the "{target}" environment.""",
+                    )
+                else:
+                    raise PlatformException(e)