dbt-platform-helper 15.3.0__py3-none-any.whl → 15.16.0__py3-none-any.whl

dbt_platform_helper/domain/config.py

@@ -7,6 +7,7 @@ from typing import Dict
 from prettytable import PrettyTable
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.constants import STANDARD_PLATFORM_SSO_ROLES
 from dbt_platform_helper.domain.versioning import AWSVersioning
 from dbt_platform_helper.domain.versioning import CopilotVersioning
 from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
@@ -140,6 +141,10 @@ class Config:
         if self.io.confirm(
             f"This command is destructive and will overwrite file contents at {file_path}. Are you sure you want to continue?"
         ):
+            self.io.info(
+                "Fetching credentials... this may take longer if you have access to many accounts."
+            )
+
             with open(aws_config_path, "w") as config_file:
                 config_file.write(AWS_CONFIG)
 
@@ -147,7 +152,9 @@ class Config:
                     config_file.write(f"[profile {account['accountName']}]\n")
                     config_file.write("sso_session = uktrade\n")
                     config_file.write(f"sso_account_id = {account['accountId']}\n")
-                    config_file.write("sso_role_name = AdministratorAccess\n")
+                    config_file.write(
+                        f"sso_role_name = {self._retrieve_role_for_aws_account(aws_sso_token=access_token, account_id=account['accountId'])}\n"
+                    )
                     config_file.write("region = eu-west-2\n")
                     config_file.write("output = json\n")
                     config_file.write("\n")
@@ -177,6 +184,28 @@ class Config:
         )
         return accounts_list
 
+    def _retrieve_role_for_aws_account(self, aws_sso_token, account_id):
+        """
+        Selects the most appropriate IAM role for a given AWS account.
+
+        Roles listed in STANDARD_PLATFORM_SSO_ROLES are preferred if available.
+        If none are found, the first available role returned by
+        sso.list_account_roles is used.
+        """
+
+        role_list = self.sso.list_account_roles(
+            access_token=aws_sso_token,
+            account_id=account_id,
+            max_results=100,
+        )
+
+        roles_retrieved = [r["roleName"] for r in role_list]
+
+        for role in STANDARD_PLATFORM_SSO_ROLES:
+            if role in roles_retrieved:
+                return role
+        return role_list[0]["roleName"]
+
     def _add_version_status_row(
         self, table: PrettyTable, header: str, version_status: VersionStatus
     ):

dbt_platform_helper/domain/database_copy.py

@@ -68,7 +68,7 @@ class DatabaseCopy:
         environment = environments.get(env)
         if not environment:
             self.io.abort_with_error(
-                f"No such environment '{env}'. Available environments are: {', '.join(environments.keys())}"
+                f"Environment '{env}' cannot be found in your account configuration in parameter store. Available environments are: {', '.join(environments.keys())}. Please check that the Terraform infrastructure for your environment is up to date."
             )
 
         env_session = environment.session

dbt_platform_helper/domain/maintenance_page.py

@@ -10,6 +10,8 @@ from typing import Union
 
 import click
 
+from dbt_platform_helper.constants import MAINTENANCE_PAGE_REASON
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
@@ -199,7 +201,13 @@ class MaintenancePage:
                         "AllowedIps",
                         next(rule_priority),
                         service_conditions,
-                        [{"Key": "service", "Value": svc.name}],
+                        [
+                            {"Key": "application", "Value": app},
+                            {"Key": "environment", "Value": env},
+                            {"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
+                            {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
+                            {"Key": "service", "Value": svc.name},
+                        ],
                     )
                     self.load_balancer.create_source_ip_rule(
                         listener_arn,
@@ -208,7 +216,13 @@ class MaintenancePage:
                         "AllowedSourceIps",
                         next(rule_priority),
                         service_conditions,
-                        [{"Key": "service", "Value": svc.name}],
+                        [
+                            {"Key": "application", "Value": app},
+                            {"Key": "environment", "Value": env},
+                            {"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
+                            {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
+                            {"Key": "service", "Value": svc.name},
+                        ],
                     )
 
                 self.load_balancer.create_header_rule(
@@ -219,7 +233,13 @@ class MaintenancePage:
                     "BypassIpFilter",
                     next(rule_priority),
                     service_conditions,
-                    [{"Key": "service", "Value": svc.name}],
+                    [
+                        {"Key": "application", "Value": app},
+                        {"Key": "environment", "Value": env},
+                        {"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
+                        {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
+                        {"Key": "service", "Value": svc.name},
+                    ],
                 )
 
                 # add to accumilating list of conditions for maintenace page rule
@@ -267,8 +287,12 @@ class MaintenancePage:
                         }
                     ],
                     tags=[
+                        {"Key": "application", "Value": app},
+                        {"Key": "environment", "Value": env},
+                        {"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
                         {"Key": "name", "Value": "MaintenancePage"},
                         {"Key": "type", "Value": template},
+                        {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
                     ],
                 )
         except Exception as e:

dbt_platform_helper/domain/pipelines.py

@@ -5,24 +5,14 @@ from shutil import rmtree
 
 from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENT_PIPELINES_KEY
-from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_OVERRIDE_KEY
 from dbt_platform_helper.constants import SUPPORTED_AWS_PROVIDER_VERSION
 from dbt_platform_helper.constants import SUPPORTED_TERRAFORM_VERSION
-from dbt_platform_helper.constants import (
-    TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
-)
-from dbt_platform_helper.constants import (
-    TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR,
-)
+from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.ecr import ECRProvider
-from dbt_platform_helper.providers.environment_variable import (
-    EnvironmentVariableProvider,
-)
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.terraform_manifest import TerraformManifestProvider
-from dbt_platform_helper.utils.application import get_application_name
 from dbt_platform_helper.utils.template import setup_templates
 
 
@@ -33,31 +23,43 @@ class Pipelines:
         terraform_manifest_provider: TerraformManifestProvider,
         ecr_provider: ECRProvider,
         get_git_remote: Callable[[], str],
-        get_codestar_arn: Callable[[str], str],
         io: ClickIOProvider = ClickIOProvider(),
         file_provider: FileProvider = FileProvider(),
-        environment_variable_provider: EnvironmentVariableProvider = None,
-        platform_helper_version_override: str = None,
+        platform_helper_versioning: PlatformHelperVersioning = None,
     ):
         self.config_provider = config_provider
         self.get_git_remote = get_git_remote
-        self.get_codestar_arn = get_codestar_arn
         self.terraform_manifest_provider = terraform_manifest_provider
         self.ecr_provider = ecr_provider
         self.io = io
         self.file_provider = file_provider
-        self.environment_variable_provider = (
-            environment_variable_provider or EnvironmentVariableProvider()
-        )
-        self.platform_helper_version_override = (
-            platform_helper_version_override
-            or self.environment_variable_provider.get(PLATFORM_HELPER_VERSION_OVERRIDE_KEY)
-        )
+        self.platform_helper_versioning = platform_helper_versioning
+
+    def _map_environment_pipeline_accounts(self, platform_config) -> list[tuple[str, str]]:
+        environment_pipelines_config = platform_config[ENVIRONMENT_PIPELINES_KEY]
+        environment_config = platform_config["environments"]
+
+        account_id_lookup = {
+            env["accounts"]["deploy"]["name"]: env["accounts"]["deploy"]["id"]
+            for env in environment_config.values()
+            if env is not None and "accounts" in env and "deploy" in env["accounts"]
+        }
+
+        accounts = set()
+
+        for config in environment_pipelines_config.values():
+            account = config.get("account")
+            deploy_account_id = account_id_lookup.get(account)
+            accounts.add((account, deploy_account_id))
+
+        return list(accounts)
 
     def generate(
         self,
         deploy_branch: str,
     ):
+        self.platform_helper_versioning.check_platform_helper_version_mismatch()
+
         platform_config = self.config_provider.load_and_validate_platform_config()
 
         has_codebase_pipelines = CODEBASE_PIPELINES_KEY in platform_config
@@ -67,28 +69,15 @@ class Pipelines:
             self.io.warn("No pipelines defined: nothing to do.")
             return
 
-        app_name = get_application_name()
-
         git_repo = self.get_git_remote()
         if not git_repo:
             self.io.abort_with_error("The current directory is not a git repository")
 
-        codestar_connection_arn = self.get_codestar_arn(app_name)
-        if codestar_connection_arn is None:
-            self.io.abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
-
         base_path = Path(".")
         copilot_pipelines_dir = base_path / f"copilot/pipelines"
 
         self._clean_pipeline_config(copilot_pipelines_dir)
 
-        platform_helper_version_for_template: str = platform_config.get("default_versions", {}).get(
-            "platform-helper"
-        )
-
-        if self.platform_helper_version_override:
-            platform_helper_version_for_template = self.platform_helper_version_override
-
         # TODO: DBTP-1965: - this whole code block/if-statement can fall away once the deploy_repository is a required key.
         deploy_repository = ""
         if "deploy_repository" in platform_config.keys():
@@ -99,28 +88,18 @@ class Pipelines:
             )
             deploy_repository = f"uktrade/{platform_config['application']}-deploy"
 
-        env_pipeline_module_source = (
-            self.environment_variable_provider.get(
-                TERRAFORM_ENVIRONMENT_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
-            )
-            or f"git::git@github.com:uktrade/platform-tools.git//terraform/environment-pipelines?depth=1&ref={platform_helper_version_for_template}"
-        )
-
         if has_environment_pipelines:
-            environment_pipelines = platform_config[ENVIRONMENT_PIPELINES_KEY]
-            accounts = {
-                config.get("account")
-                for config in environment_pipelines.values()
-                if "account" in config
-            }
+            accounts = self._map_environment_pipeline_accounts(platform_config)
 
-            for account in accounts:
+            for account_name, account_id in accounts:
                 self._generate_terraform_environment_pipeline_manifest(
                     platform_config["application"],
                     deploy_repository,
-                    account,
-                    env_pipeline_module_source,
+                    account_name,
+                    self.platform_helper_versioning.get_environment_pipeline_modules_source(),
                     deploy_branch,
+                    account_id,
+                    self.platform_helper_versioning.get_pinned_version(),
                 )
 
         if has_codebase_pipelines:
@@ -136,19 +115,12 @@ class Pipelines:
                 if repo in ecrs_already_provisioned
             }
 
-            codebase_pipeline_module_source = (
-                self.environment_variable_provider.get(
-                    TERRAFORM_CODEBASE_PIPELINES_MODULE_SOURCE_OVERRIDE_ENV_VAR
-                )
-                or f"git::git@github.com:uktrade/platform-tools.git//terraform/codebase-pipelines?depth=1&ref={platform_helper_version_for_template}"
-            )
-
             self.terraform_manifest_provider.generate_codebase_pipeline_config(
                 platform_config,
-                platform_helper_version_for_template,
+                self.platform_helper_versioning.get_template_version(),
                 ecrs_that_need_importing,
                 deploy_repository,
-                codebase_pipeline_module_source,
+                self.platform_helper_versioning.get_codebase_pipeline_modules_source(),
             )
 
     def _clean_pipeline_config(self, pipelines_dir: Path):
@@ -163,6 +135,8 @@ class Pipelines:
         aws_account: str,
         module_source: str,
         deploy_branch: str,
+        aws_account_id: str,
+        pinned_version: str,
     ):
         env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
 
@@ -175,6 +149,8 @@ class Pipelines:
                 "deploy_branch": deploy_branch,
                 "terraform_version": SUPPORTED_TERRAFORM_VERSION,
                 "aws_provider_version": SUPPORTED_AWS_PROVIDER_VERSION,
+                "deploy_account_id": aws_account_id,
+                "pinned_version": pinned_version,
             }
         )
 

dbt_platform_helper/domain/secrets.py

@@ -0,0 +1,279 @@
+import botocore
+
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
+from dbt_platform_helper.constants import MANAGED_BY_PLATFORM_TERRAFORM
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import Parameter
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.utils.application import load_application
+
+
+class Secrets:
+
+    def __init__(
+        self,
+        load_application=load_application,
+        io: ClickIOProvider = ClickIOProvider(),
+        parameter_store_provider: ParameterStore = ParameterStore,
+    ):
+        self.load_application_fn = load_application
+        self.application = None
+        self.io = io
+        self.parameter_store_provider: ParameterStore = parameter_store_provider
+
+    def _check_ssm_write_access(self, accounts):
+        """Check access."""
+        no_access = []
+        for account, session in accounts.items():
+            sts = session.client("sts")
+            iam = session.client("iam")
+
+            sts_arn = sts.get_caller_identity()["Arn"]
+            role_name = sts_arn.split("/")[1]
+
+            role_arn = (
+                f"arn:aws:iam::{account}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}"
+            )
+            response = iam.simulate_principal_policy(
+                PolicySourceArn=role_arn,
+                ActionNames=[
+                    "ssm:PutParameter",
+                ],
+                ContextEntries=[
+                    {
+                        "ContextKeyName": "aws:RequestedRegion",
+                        "ContextKeyValues": [
+                            "eu-west-2",
+                        ],
+                        "ContextKeyType": "string",
+                    }
+                ],
+            )["EvaluationResults"]
+
+            has_access = [
+                account for eval_result in response if eval_result["EvalDecision"] == "allowed"
+            ]
+
+            if not has_access:
+                no_access.append(account)
+
+        if no_access:
+            account_ids = "', '".join(no_access)
+            raise PlatformException(
+                f"You do not have AWS Parameter Store write access to the following AWS accounts: '{account_ids}'"
+            )
+
+    def _check_for_existing_params(self, get_secret_name):
+        found_params = []
+        for _, environment in self.application.environments.items():
+            parameter_store: ParameterStore = self.parameter_store_provider(
+                environment.session.client("ssm")
+            )
+            try:
+                param = parameter_store.get_ssm_parameter_by_name(get_secret_name(environment.name))
+                if param:
+                    found_params.append(environment.name)
+            except botocore.exceptions.ClientError as error:
+                if error.response["Error"]["Code"] == "ParameterNotFound":
+                    pass
+                else:
+                    raise PlatformException(error)
+
+        return found_params
+
+    def create(self, app_name, name, overwrite):
+        self.application = (
+            self.load_application_fn(app_name) if not self.application else self.application
+        )
+
+        accounts = {}
+        for _, environment in self.application.environments.items():
+            if environment.account_id not in accounts:
+                accounts[environment.account_id] = environment.session
+
+        self._check_ssm_write_access(accounts)
+
+        get_secret_name = lambda env: f"/platform/{app_name}/{env}/secrets/{name.upper()}"
+        found_params = self._check_for_existing_params(get_secret_name)
+
+        if overwrite is False and found_params:
+            envs = "', '".join(found_params)
+            raise PlatformException(
+                f"AWS Parameter Store secret '{name.upper()}' already exists for the following environments: '{envs}'. \nUse the --overwrite flag to replacing existing secret values."
+            )
+
+        values = {}
+        for _, environment in self.application.environments.items():
+            value = self.io.input(
+                f"Please enter value for secret '{name.upper()}' in environment '{environment.name}'",
+                hide_input=True,
+            )
+            values[environment.name] = value
+
+        for environment_name, secret_value in values.items():
+
+            environment = self.application.environments[environment_name]
+            parameter_store: ParameterStore = self.parameter_store_provider(
+                environment.session.client("ssm")
+            )
+
+            data_dict = dict(
+                Name=get_secret_name(environment.name),
+                Value=secret_value,
+                Overwrite=overwrite,
+                Type="SecureString",
+                Tags=[
+                    {"Key": "application", "Value": app_name},
+                    {"Key": "environment", "Value": environment.name},
+                    {"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
+                ],
+            )
+
+            # If in found params we are overwriting
+            if overwrite and environment_name in found_params:
+                data_dict["Overwrite"] = True
+                del data_dict["Tags"]
+            self.io.debug(
+                f"Creating AWS Parameter Store secret {get_secret_name(environment.name)} ..."
+            )
+            parameter_store.put_parameter(data_dict)
+            self.io.debug(
+                f"Successfully created AWS Parameter Store secret {get_secret_name(environment.name)}"
+            )
+
+        self.io.info(
+            "\nTo check or update your secrets, log into your AWS account via the Console and visit the Parameter Store https://eu-west-2.console.aws.amazon.com/systems-manager/parameters/\n"
+            "You can attach secrets into ECS container by adding them to the `secrets` section of your 'service-config.yml' file."
+        )
+
+        self.io.info(
+            message=f"```\nsecrets:\n\t{name.upper()}: /platform/${{PLATFORM_APPLICATION_NAME}}/${{PLATFORM_ENVIRONMENT_NAME}}/secrets/{name.upper()}\n```",
+            fg="cyan",
+            bold=True,
+        )
+
+    def __has_access(self, env, actions=["ssm:PutParameter"], access_type="write"):
+        sts_arn = env.session.client("sts").get_caller_identity()["Arn"]
+        role_name = sts_arn.split("/")[1]
+
+        role_arn = f"arn:aws:iam::{env.account_id}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}"
+        response = env.session.client("iam").simulate_principal_policy(
+            PolicySourceArn=role_arn,
+            ActionNames=actions,
+            ContextEntries=[
+                {
+                    "ContextKeyName": "aws:RequestedRegion",
+                    "ContextKeyValues": [
+                        "eu-west-2",
+                    ],
+                    "ContextKeyType": "string",
+                }
+            ],
+        )["EvaluationResults"]
+        has_access = [
+            env.account_id for eval_result in response if eval_result["EvalDecision"] == "allowed"
+        ]
+
+        if not has_access:
+            raise PlatformException(
+                f"You do not have AWS Parameter Store {access_type} access to the following AWS accounts: '{env.account_id}'"
+            )
+
+    def copy(self, app_name: str, source: str, target: str):
+        """Copy AWS Parameter Store secrets from one environment into
+        another."""
+
+        self.application = (
+            self.load_application_fn(app_name) if not self.application else self.application
+        )
+
+        if not self.application.environments.get(target, ""):
+            raise PlatformException(
+                f"Environment '{target}' not found for application '{app_name}'."
+            )
+        elif not self.application.environments.get(source, ""):
+            raise PlatformException(
+                f"Environment '{source}' not found for application '{app_name}'."
+            )
+
+        source_env = self.application.environments.get(source)
+        target_env = self.application.environments.get(target)
+
+        self.__has_access(source_env, actions=["ssm:GetParameter"], access_type="read")
+        self.__has_access(target_env)
+
+        prod_account_id = self.application.environments["prod"].account_id
+        if (
+            self.application.environments[source].account_id == prod_account_id
+            and self.application.environments[target].account_id != prod_account_id
+        ):
+            raise PlatformException(
+                f"Cannot transfer secrets out from '{source}' in the prod account '{prod_account_id}'"
+                f" to '{target}' in '{self.application.environments[target].account_id}'"
+            )
+
+        parameter_store: ParameterStore = self.parameter_store_provider(
+            source_env.session.client("ssm"), with_model=True
+        )
+
+        target_parameter_store: ParameterStore = self.parameter_store_provider(
+            target_env.session.client("ssm"), with_model=True
+        )
+
+        copilot_secrets: list[Parameter] = parameter_store.get_ssm_parameters_by_path(
+            f"/copilot/{app_name}/{source}/secrets", add_tags=True
+        )
+        platform_secrets: list[Parameter] = parameter_store.get_ssm_parameters_by_path(
+            f"/platform/{app_name}/{source}/secrets", add_tags=True
+        )
+
+        secrets = copilot_secrets + platform_secrets
+
+        for secret in secrets:
+            secret.name = secret.name.replace(f"/{source}/", f"/{target}/")
+
+            if (
+                "AWS_" in secret.name
+                or secret.tags.get("managed-by", "") == MANAGED_BY_PLATFORM_TERRAFORM
+                or secret.tags.get("managed-by", "")
+                == "Terraform"  # SSM params POSTGRES_APPLICATION_USER and POSTGRES_READ_ONLY_USER are tagged differently from the rest
+            ):
+                message = f"Skipping AWS Parameter Store secret {secret.name}"
+                if secret.tags.get("managed-by", ""):
+                    managed_by = secret.tags["managed-by"]
+                    message += f" with managed-by: {managed_by}"
+                self.io.debug(message)
+                continue
+
+            secret.tags["application"] = app_name
+            secret.tags["environment"] = target
+            secret.tags["managed-by"] = MANAGED_BY_PLATFORM
+            secret.tags["copied-from"] = source
+
+            if secret.name.startswith("/copilot/"):
+                secret.tags["copilot-environment"] = target
+
+            data_dict = dict(
+                Name=secret.name,
+                Value=secret.value,
+                Overwrite=False,
+                Type=secret.type,
+                Description=f"Copied from {source} environment.",
+                Tags=secret.tags_to_list(),
+            )
+            self.io.debug(f"Creating AWS Parameter Store secret {secret.name} ...")
+
+            try:
+                target_parameter_store.put_parameter(data_dict)
+                secret_name = secret.name.split("/")[-1]
+                self.io.info(
+                    f"Secret {secret_name} was successfully copied from the '{source} environment to '{target}'"
+                )
+            except botocore.exceptions.ClientError as e:
+                if e.response["Error"]["Code"] == "ParameterAlreadyExists":
+                    self.io.warn(
+                        f"""The "{secret.name.split("/")[-1]}" parameter already exists for the "{target}" environment.""",
+                    )
+                else:
+                    raise PlatformException(e)