dbt-platform-helper 13.2.0__tar.gz → 13.3.0__tar.gz

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 13.2.0
+Version: 13.3.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: Jinja2 (==3.1.5)
+Requires-Dist: Jinja2 (==3.1.6)
 Requires-Dist: PyYAML (==6.0.1)
 Requires-Dist: aiohttp (>=3.8.4,<4.0.0)
 Requires-Dist: boto3 (>=1.28.24,<2.0.0)

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/dbt_platform_helper/commands/codebase.py

@@ -4,6 +4,8 @@ from dbt_platform_helper.domain.codebase import Codebase
 from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 
 
@@ -17,7 +19,7 @@ def codebase():
 def prepare():
     """Sets up an application codebase for use within a DBT platform project."""
     try:
-        Codebase().prepare()
+        Codebase(ParameterStore(get_aws_session_or_abort().client("ssm"))).prepare()
     except PlatformException as err:
         ClickIOProvider().abort_with_error(str(err))
 
@@ -33,7 +35,7 @@ def prepare():
 def list(app, with_images):
     """List available codebases for the application."""
     try:
-        Codebase().list(app, with_images)
+        Codebase(ParameterStore(get_aws_session_or_abort().client("ssm"))).list(app, with_images)
     except PlatformException as err:
         ClickIOProvider().abort_with_error(str(err))
 
@@ -49,7 +51,9 @@ def list(app, with_images):
 def build(app, codebase, commit):
     """Trigger a CodePipeline pipeline based build."""
     try:
-        Codebase().build(app, codebase, commit)
+        Codebase(ParameterStore(get_aws_session_or_abort().client("ssm"))).build(
+            app, codebase, commit
+        )
     except PlatformException as err:
         ClickIOProvider().abort_with_error(str(err))
 
@@ -65,6 +69,8 @@ def build(app, codebase, commit):
 @click.option("--commit", help="GitHub commit hash", required=True)
 def deploy(app, env, codebase, commit):
     try:
-        Codebase().deploy(app, env, codebase, commit)
+        Codebase(ParameterStore(get_aws_session_or_abort().client("ssm"))).deploy(
+            app, env, codebase, commit
+        )
     except PlatformException as err:
         ClickIOProvider().abort_with_error(str(err))

dbt_platform_helper-13.3.0/dbt_platform_helper/commands/config.py

@@ -0,0 +1,38 @@
+import click
+
+from dbt_platform_helper.domain.config import Config
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.aws.sso_auth import SSOAuthProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+
+
+@click.group(cls=ClickDocOptGroup)
+def config():
+    """Perform actions on configuration files."""
+
+
+@config.command()
+def validate():
+    """Validate deployment or application configuration."""
+    try:
+        Config().validate()
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))
+
+
+@config.command()
+@click.option("--file-path", "-fp", default="~/.aws/config")
+def aws(file_path):
+    """
+    Writes a local config file containing all the AWS profiles to which the
+    logged in user has access.
+
+    If no `--file-path` is specified, defaults to `~/.aws/config`.
+    """
+    try:
+        session = get_aws_session_or_abort()
+        Config(sso=SSOAuthProvider(session)).generate_aws(file_path)
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/dbt_platform_helper/commands/copilot.py

@@ -10,13 +10,10 @@ from dbt_platform_helper.providers.config_validator import ConfigValidator
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.providers.kms import KMSProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
 from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 
-# TODOs
-# Figure out a pattern for copilot templating and the new copilot domain - probably a lot of overlap here that really belongs in the copilottemplating domain instead (atleast whatever is concerned with "templating")
-# Check for E2E test coverage.
-
 
 @click.group(chain=True, cls=ClickDocOptGroup)
 def copilot():
@@ -28,8 +25,15 @@ def make_addons():
     """Generate addons CloudFormation for each environment."""
     try:
         session = get_aws_session_or_abort()
+        parameter_provider = ParameterStore(session.client("ssm"))
         config_provider = ConfigProvider(ConfigValidator())
-        kms_provider = KMSProvider(session.client("kms"))
-        Copilot(config_provider, FileProvider(), CopilotTemplating(), kms_provider).make_addons()
+        Copilot(
+            config_provider,
+            parameter_provider,
+            FileProvider(),
+            CopilotTemplating(),
+            KMSProvider,
+            session,
+        ).make_addons()
     except Exception as err:
         ClickIOProvider().abort_with_error(str(err))

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/dbt_platform_helper/commands/database.py

@@ -2,6 +2,8 @@ import click
 
 from dbt_platform_helper.commands.environment import AVAILABLE_TEMPLATES
 from dbt_platform_helper.domain.database_copy import DatabaseCopy
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.io import ClickIOProvider
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 
 
@@ -38,9 +40,11 @@ def database():
 )
 def dump(app, from_env, database, from_vpc, filename):
     """Dump a database into an S3 bucket."""
-    data_copy = DatabaseCopy(app, database)
-    data_copy.dump(from_env, from_vpc, filename)
-    # Todo: Catch expected errors and output message
+    try:
+        data_copy = DatabaseCopy(app, database)
+        data_copy.dump(from_env, from_vpc, filename)
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @database.command(name="load")
@@ -68,9 +72,11 @@ def dump(app, from_env, database, from_vpc, filename):
 )
 def load(app, to_env, database, to_vpc, auto_approve, filename):
     """Load a database from an S3 bucket."""
-    data_copy = DatabaseCopy(app, database, auto_approve)
-    data_copy.load(to_env, to_vpc, filename)
-    # Todo: Catch expected errors and output message
+    try:
+        data_copy = DatabaseCopy(app, database, auto_approve)
+        data_copy.load(to_env, to_vpc, filename)
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))
 
 
 @database.command(name="copy")
@@ -120,6 +126,8 @@ def copy(
     no_maintenance_page,
 ):
     """Copy a database between environments."""
-    data_copy = DatabaseCopy(app, database, auto_approve)
-    data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
-    # Todo: Catch expected errors and output message
+    try:
+        data_copy = DatabaseCopy(app, database, auto_approve)
+        data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
+    except PlatformException as err:
+        ClickIOProvider().abort_with_error(str(err))

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/dbt_platform_helper/commands/environment.py

@@ -94,9 +94,8 @@ def generate(name):
 def generate_terraform(name, terraform_platform_modules_version):
     click_io = ClickIOProvider()
     try:
-        # TODO = pass the session to ConfigValidator
-        get_aws_session_or_abort()
-        config_provider = ConfigProvider(ConfigValidator())
+        session = get_aws_session_or_abort()
+        config_provider = ConfigProvider(ConfigValidator(session=session))
         TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
     except PlatformException as err:
         click_io.abort_with_error(str(err))

{dbt_platform_helper-13.2.0 → dbt_platform_helper-13.3.0}/dbt_platform_helper/domain/codebase.py

@@ -11,6 +11,7 @@ from boto3 import Session
 from dbt_platform_helper.platform_exception import PlatformException
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.parameter_store import ParameterStore
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import (
     ApplicationEnvironmentNotFoundException,
@@ -32,6 +33,7 @@ from dbt_platform_helper.utils.template import setup_templates
 class Codebase:
     def __init__(
         self,
+        parameter_provider: ParameterStore,
         io: ClickIOProvider = ClickIOProvider(),
         load_application: Callable[[str], Application] = load_application,
         get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
@@ -50,6 +52,7 @@ class Codebase:
         check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
         run_subprocess: Callable[[str], str] = subprocess.run,
     ):
+        self.parameter_provider = parameter_provider
         self.io = io
         self.load_application = load_application
         self.get_aws_session_or_abort = get_aws_session_or_abort
@@ -190,9 +193,8 @@ class Codebase:
         """List available codebases for the application."""
         session = self.get_aws_session_or_abort()
         application = self.load_application(app, session)
-        ssm_client = session.client("ssm")
         ecr_client = session.client("ecr")
-        codebases = self.__get_codebases(application, ssm_client)
+        codebases = self.__get_codebases(application, session.client("ssm"))
 
         self.io.info("The following codebases are available:")
 
@@ -209,11 +211,9 @@ class Codebase:
         self.io.info("")
 
     def __get_codebases(self, application, ssm_client):
-        parameters = ssm_client.get_parameters_by_path(
-            Path=f"/copilot/applications/{application.name}/codebases",
-            Recursive=True,
-        )["Parameters"]
-
+        parameters = self.parameter_provider.get_ssm_parameters_by_path(
+            f"/copilot/applications/{application.name}/codebases"
+        )
         codebases = [json.loads(p["Value"]) for p in parameters]
 
         if not codebases:

dbt_platform_helper-13.3.0/dbt_platform_helper/domain/config.py

@@ -0,0 +1,345 @@
+import os
+import re
+import webbrowser
+from pathlib import Path
+from typing import Dict
+
+from prettytable import PrettyTable
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.versioning import AWSVersioning
+from dbt_platform_helper.domain.versioning import CopilotVersioning
+from dbt_platform_helper.domain.versioning import PlatformHelperVersioning
+from dbt_platform_helper.platform_exception import PlatformException
+from dbt_platform_helper.providers.aws.sso_auth import SSOAuthProvider
+from dbt_platform_helper.providers.io import ClickIOProvider
+from dbt_platform_helper.providers.semantic_version import (
+    IncompatibleMajorVersionException,
+)
+from dbt_platform_helper.providers.semantic_version import SemanticVersion
+from dbt_platform_helper.providers.validation import ValidationException
+from dbt_platform_helper.providers.version_status import PlatformHelperVersionStatus
+from dbt_platform_helper.providers.version_status import VersionStatus
+
+yes = "\033[92m✔\033[0m"
+no = "\033[91m✖\033[0m"
+maybe = "\033[93m?\033[0m"
+
+RECOMMENDATIONS = {
+    "dbt-platform-helper-upgrade": (
+        "Upgrade dbt-platform-helper to version {version} `pip install "
+        "--upgrade dbt-platform-helper=={version}`."
+    ),
+    "dbt-platform-helper-upgrade-note": (
+        "Post upgrade, run `platform-helper copilot make-addons` to " "update your addon templates."
+    ),
+    "generic-tool-upgrade": "Upgrade {tool} to version {version}.",
+    "install-copilot": "Install AWS Copilot https://aws.github.io/copilot-cli/",
+    "install-aws": "Install AWS CLI https://aws.amazon.com/cli/",
+}
+
+AWS_CONFIG = """
+#
+# uktrade
+#
+
+[sso-session uktrade]
+sso_start_url = https://uktrade.awsapps.com/start#/
+sso_region = eu-west-2
+sso_registration_scopes = sso:account:access
+
+[default]
+sso_session = uktrade
+region = eu-west-2
+output = json
+
+"""
+
+
+class NoDeploymentRepoConfigException(PlatformException):
+    def __init__(self):
+        super().__init__("Could not find a deployment repository, no checks to run.")
+
+
+# TODO move to generic location so it can be reused
+class NoPlatformConfigException(PlatformException):
+    def __init__(self):
+        super().__init__(
+            f"`platform-config.yml` is missing. "
+            "Please check it exists and you are in the root directory of your deployment project."
+        )
+
+
+class Config:
+
+    def __init__(
+        self,
+        io: ClickIOProvider = ClickIOProvider(),
+        platform_helper_versioning: PlatformHelperVersioning = PlatformHelperVersioning(),
+        aws_versioning: AWSVersioning = AWSVersioning(),
+        copilot_versioning: CopilotVersioning = CopilotVersioning(),
+        sso: SSOAuthProvider = None,
+    ):
+        self.oidc_app = None
+        self.io = io
+        self.platform_helper_versioning = platform_helper_versioning
+        self.aws_versioning = aws_versioning
+        self.copilot_versioning = copilot_versioning
+        self.sso = sso or SSOAuthProvider()
+        self.SSO_START_URL = "https://uktrade.awsapps.com/start"
+
+    def validate(self):
+        if not Path("copilot").exists():
+            raise NoDeploymentRepoConfigException()
+        if not Path(PLATFORM_CONFIG_FILE).exists():
+            raise NoPlatformConfigException()
+
+        self.io.debug("\nDetected a deployment repository\n")
+        platform_helper_version_status = self.platform_helper_versioning._get_version_status(
+            include_project_versions=True
+        )
+        self.io.process_messages(platform_helper_version_status.validate())
+        aws_version_status = self.aws_versioning.get_version_status()
+        copilot_version_status = self.copilot_versioning.get_version_status()
+
+        self._check_tool_versions(
+            platform_helper_version_status, aws_version_status, copilot_version_status
+        )
+
+        compatible = self._check_addon_versions(platform_helper_version_status)
+
+        exit(0 if compatible else 1)
+
+    def generate_aws(self, file_path):
+        self.oidc_app = self._create_oidc_application()
+        verification_url, device_code = self._get_device_code(self.oidc_app)
+
+        if self.io.confirm(
+            "You are about to be redirected to a verification page. You will need to complete sign-in before returning to the command line. Do you want to continue?",
+        ):
+            webbrowser.open(verification_url)
+
+        if self.io.confirm(
+            "Have you completed the sign-in process in your browser?",
+        ):
+            access_token = self.sso.create_access_token(
+                client_id=self.oidc_app[0],
+                client_secret=self.oidc_app[1],
+                device_code=device_code,
+            )
+
+        aws_config_path = os.path.expanduser(file_path)
+
+        if self.io.confirm(
+            f"This command is destructive and will overwrite file contents at {file_path}. Are you sure you want to continue?"
+        ):
+            with open(aws_config_path, "w") as config_file:
+                config_file.write(AWS_CONFIG)
+
+                for account in self._retrieve_aws_accounts(access_token):
+                    config_file.write(f"[profile {account['accountName']}]\n")
+                    config_file.write("sso_session = uktrade\n")
+                    config_file.write(f"sso_account_id = {account['accountId']}\n")
+                    config_file.write("sso_role_name = AdministratorAccess\n")
+                    config_file.write("region = eu-west-2\n")
+                    config_file.write("output = json\n")
+                    config_file.write("\n")
+
+    def _create_oidc_application(self):
+        self.io.debug("Creating temporary AWS SSO OIDC application")
+        client_id, client_secret = self.sso.register(
+            client_name="platform-helper",
+            client_type="public",
+        )
+        return client_id, client_secret
+
+    def _get_device_code(self, oidc_application):
+        self.io.debug("Initiating device code flow")
+        url, device_code = self.sso.start_device_authorization(
+            client_id=oidc_application[0],
+            client_secret=oidc_application[1],
+            start_url=self.SSO_START_URL,
+        )
+
+        return url, device_code
+
+    def _retrieve_aws_accounts(self, aws_sso_token):
+        accounts_list = self.sso.list_accounts(
+            access_token=aws_sso_token,
+            max_results=100,
+        )
+        return accounts_list
+
+    def _add_version_status_row(
+        self, table: PrettyTable, header: str, version_status: VersionStatus
+    ):
+        table.add_row(
+            [
+                header,
+                str(version_status.installed),
+                str(version_status.latest),
+                no if version_status.is_outdated() else yes,
+            ]
+        )
+
+    def _check_tool_versions(
+        self,
+        platform_helper_version_status: PlatformHelperVersionStatus,
+        aws_version_status: VersionStatus,
+        copilot_version_status: VersionStatus,
+    ):
+        self.io.debug("Checking tooling versions...")
+
+        recommendations = {}
+
+        if copilot_version_status.installed is None:
+            recommendations["install-copilot"] = RECOMMENDATIONS["install-copilot"]
+
+        if aws_version_status.installed is None:
+            recommendations["install-aws"] = RECOMMENDATIONS["install-aws"]
+
+        tool_versions_table = PrettyTable()
+        tool_versions_table.field_names = [
+            "Tool",
+            "Local version",
+            "Released version",
+            "Running latest?",
+        ]
+        tool_versions_table.align["Tool"] = "l"
+
+        self._add_version_status_row(tool_versions_table, "aws", aws_version_status)
+        self._add_version_status_row(tool_versions_table, "copilot", copilot_version_status)
+        self._add_version_status_row(
+            tool_versions_table, "dbt-platform-helper", platform_helper_version_status
+        )
+
+        self.io.info(tool_versions_table)
+
+        if aws_version_status.is_outdated() and "install-aws" not in recommendations:
+            recommendations["aws-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
+                tool="AWS CLI",
+                version=str(aws_version_status.latest),
+            )
+
+        if copilot_version_status.is_outdated() and "install-copilot" not in recommendations:
+            recommendations["copilot-upgrade"] = RECOMMENDATIONS["generic-tool-upgrade"].format(
+                tool="AWS Copilot",
+                version=str(copilot_version_status.latest),
+            )
+
+        if platform_helper_version_status.is_outdated():
+            recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
+                "dbt-platform-helper-upgrade"
+            ].format(version=str(platform_helper_version_status.latest))
+            recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
+                "dbt-platform-helper-upgrade-note"
+            ]
+
+        self._render_recommendations(recommendations)
+
+    def _check_addon_versions(self, platform_helper_versions: PlatformHelperVersionStatus) -> bool:
+
+        self.io.debug("Checking addons templates versions...")
+
+        compatible = True
+        recommendations = {}
+
+        local_version = platform_helper_versions.installed
+        latest_release = platform_helper_versions.latest
+
+        addons_templates_table = PrettyTable()
+        addons_templates_table.field_names = [
+            "Addons Template File",
+            "Generated with",
+            "Compatible with local?",
+            "Compatible with latest?",
+        ]
+        addons_templates_table.align["Addons Template File"] = "l"
+
+        addons_templates = list(Path("./copilot").glob("**/addons/*"))
+        # Sort by template file path
+        addons_templates.sort(key=lambda e: str(e))
+        # Bring environment addons to the top
+        addons_templates.sort(key=lambda e: "environments/" not in str(e))
+
+        for template_file in addons_templates:
+            generated_with_version = maybe
+            local_compatible_symbol = yes
+            latest_compatible_symbol = yes
+
+            generated_with_version = None
+
+            try:
+                generated_with_version = self.__get_template_generated_with_version(
+                    str(template_file.resolve())
+                )
+            except ValidationException:
+                local_compatible_symbol = maybe
+                compatible = False
+                recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
+                    "dbt-platform-helper-upgrade"
+                ].format(version=latest_release)
+                recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
+                    "dbt-platform-helper-upgrade-note"
+                ]
+
+            try:
+                local_version.validate_compatibility_with(generated_with_version)
+            except IncompatibleMajorVersionException:
+                local_compatible_symbol = no
+                compatible = False
+                recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
+                    "dbt-platform-helper-upgrade"
+                ].format(version=latest_release)
+                recommendations["dbt-platform-helper-upgrade-note"] = RECOMMENDATIONS[
+                    "dbt-platform-helper-upgrade-note"
+                ]
+            except ValidationException:
+                local_compatible_symbol = maybe
+                compatible = False
+
+            try:
+                latest_release.validate_compatibility_with(generated_with_version)
+            except IncompatibleMajorVersionException:
+                latest_compatible_symbol = no
+                compatible = False
+            except ValidationException:
+                latest_compatible_symbol = maybe
+                compatible = False
+
+            addons_templates_table.add_row(
+                [
+                    template_file.relative_to("."),
+                    (maybe if latest_compatible_symbol is maybe else str(generated_with_version)),
+                    local_compatible_symbol,
+                    latest_compatible_symbol,
+                ]
+            )
+
+        self.io.info(addons_templates_table)
+        self._render_recommendations(recommendations)
+
+        return compatible
+
+    def _render_recommendations(self, recommendations: Dict[str, str]):
+        if recommendations:
+            self.io.info("\nRecommendations:\n", bold=True)
+
+            for name, recommendation in recommendations.items():
+                if name.endswith("-note"):
+                    continue
+                self.io.info(f"  - {recommendation}")
+                if recommendations.get(f"{name}-note", False):
+                    self.io.info(f"    {recommendations.get(f'{name}-note')}")
+
+            self.io.info("")
+
+    def __get_template_generated_with_version(self, template_file_path: str) -> SemanticVersion:
+        try:
+            template_contents = Path(template_file_path).read_text()
+            template_version = re.search(
+                r"# Generated by platform-helper ([v.\-0-9]+)", template_contents
+            ).group(1)
+            return SemanticVersion.from_string(template_version)
+        except (IndexError, AttributeError):
+            raise ValidationException(f"Template {template_file_path} has no version information")